diff options
Diffstat (limited to 'compiler')
158 files changed, 10362 insertions, 33333 deletions
diff --git a/compiler/Android.bp b/compiler/Android.bp index 307a42cbba..b721d210fe 100644 --- a/compiler/Android.bp +++ b/compiler/Android.bp @@ -106,21 +106,15 @@ art_cc_defaults { "jni/quick/arm/calling_convention_arm.cc", "linker/arm/relative_patcher_arm_base.cc", "linker/arm/relative_patcher_thumb2.cc", - "optimizing/code_generator_arm.cc", - "optimizing/code_generator_vector_arm.cc", "optimizing/code_generator_arm_vixl.cc", "optimizing/code_generator_vector_arm_vixl.cc", - "optimizing/dex_cache_array_fixups_arm.cc", "optimizing/instruction_simplifier_arm.cc", "optimizing/instruction_simplifier_shared.cc", - "optimizing/intrinsics_arm.cc", "optimizing/intrinsics_arm_vixl.cc", "optimizing/nodes_shared.cc", "optimizing/scheduler_arm.cc", - "utils/arm/assembler_arm.cc", "utils/arm/assembler_arm_vixl.cc", - "utils/arm/assembler_thumb2.cc", - "utils/arm/jni_macro_assembler_arm.cc", + "utils/arm/constants_arm.cc", "utils/arm/jni_macro_assembler_arm_vixl.cc", "utils/arm/managed_register_arm.cc", ], @@ -145,7 +139,6 @@ art_cc_defaults { "linker/mips/relative_patcher_mips.cc", "optimizing/code_generator_mips.cc", "optimizing/code_generator_vector_mips.cc", - "optimizing/dex_cache_array_fixups_mips.cc", "optimizing/intrinsics_mips.cc", "optimizing/pc_relative_fixups_mips.cc", "utils/mips/assembler_mips.cc", @@ -342,6 +335,7 @@ art_cc_test { "image_test.cc", "image_write_read_test.cc", "jni/jni_compiler_test.cc", + "linker/method_bss_mapping_encoder_test.cc", "linker/multi_oat_relative_patcher_test.cc", "linker/output_stream_test.cc", "oat_test.cc", @@ -357,6 +351,7 @@ art_cc_test { "optimizing/live_interval_test.cc", "optimizing/loop_optimization_test.cc", "optimizing/nodes_test.cc", + "optimizing/nodes_vector_test.cc", "optimizing/parallel_move_test.cc", "optimizing/pretty_printer_test.cc", "optimizing/reference_type_propagation_test.cc", @@ -365,7 +360,7 @@ art_cc_test { "optimizing/ssa_test.cc", "optimizing/stack_map_test.cc", "optimizing/suspend_check_test.cc", - "utils/atomic_method_ref_map_test.cc", + "utils/atomic_dex_ref_map_test.cc", "utils/dedupe_set_test.cc", "utils/intrusive_forward_list_test.cc", "utils/string_reference_test.cc", @@ -430,13 +425,20 @@ art_cc_test { shared_libs: [ "libartd-compiler", - "libartd-simulator", "libvixld-arm", "libvixld-arm64", "libbacktrace", "libnativeloader", ], + + target: { + host: { + shared_libs: [ + "libartd-simulator", + ], + }, + }, } art_cc_test { @@ -448,7 +450,6 @@ art_cc_test { codegen: { arm: { srcs: [ - "utils/arm/assembler_thumb2_test.cc", "utils/assembler_thumb_test.cc", ], }, @@ -456,6 +457,7 @@ art_cc_test { srcs: [ "optimizing/emit_swap_mips_test.cc", "utils/mips/assembler_mips_test.cc", + "utils/mips/assembler_mips32r5_test.cc", "utils/mips/assembler_mips32r6_test.cc", ], }, diff --git a/compiler/common_compiler_test.cc b/compiler/common_compiler_test.cc index a1ee68faeb..07bfe31e14 100644 --- a/compiler/common_compiler_test.cc +++ b/compiler/common_compiler_test.cc @@ -19,6 +19,7 @@ #include "arch/instruction_set_features.h" #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "class_linker.h" #include "compiled_method.h" @@ -166,8 +167,8 @@ void CommonCompilerTest::SetUp() { instruction_set_features_ = InstructionSetFeatures::FromCppDefines(); runtime_->SetInstructionSet(instruction_set); - for (int i = 0; i < Runtime::kLastCalleeSaveType; i++) { - Runtime::CalleeSaveType type = Runtime::CalleeSaveType(i); + for (uint32_t i = 0; i < static_cast<uint32_t>(CalleeSaveType::kLastCalleeSaveType); ++i) { + CalleeSaveType type = CalleeSaveType(i); if (!runtime_->HasCalleeSaveMethod(type)) { runtime_->SetCalleeSaveMethod(runtime_->CreateCalleeSaveMethod(), type); } @@ -206,8 +207,10 @@ void CommonCompilerTest::SetUpRuntimeOptions(RuntimeOptions* options) { compiler_options_.reset(new CompilerOptions); verification_results_.reset(new VerificationResults(compiler_options_.get())); - callbacks_.reset(new QuickCompilerCallbacks(verification_results_.get(), - CompilerCallbacks::CallbackMode::kCompileApp)); + QuickCompilerCallbacks* callbacks = + new QuickCompilerCallbacks(CompilerCallbacks::CallbackMode::kCompileApp); + callbacks->SetVerificationResults(verification_results_.get()); + callbacks_.reset(callbacks); } Compiler::Kind CommonCompilerTest::GetCompilerKind() const { @@ -264,8 +267,8 @@ void CommonCompilerTest::CompileDirectMethod(Handle<mirror::ClassLoader> class_l mirror::Class* klass = class_linker_->FindClass(self, class_descriptor.c_str(), class_loader); CHECK(klass != nullptr) << "Class not found " << class_name; auto pointer_size = class_linker_->GetImagePointerSize(); - ArtMethod* method = klass->FindDirectMethod(method_name, signature, pointer_size); - CHECK(method != nullptr) << "Direct method not found: " + ArtMethod* method = klass->FindClassMethod(method_name, signature, pointer_size); + CHECK(method != nullptr && method->IsDirect()) << "Direct method not found: " << class_name << "." << method_name << signature; CompileMethod(method); } @@ -278,8 +281,8 @@ void CommonCompilerTest::CompileVirtualMethod(Handle<mirror::ClassLoader> class_ mirror::Class* klass = class_linker_->FindClass(self, class_descriptor.c_str(), class_loader); CHECK(klass != nullptr) << "Class not found " << class_name; auto pointer_size = class_linker_->GetImagePointerSize(); - ArtMethod* method = klass->FindVirtualMethod(method_name, signature, pointer_size); - CHECK(method != nullptr) << "Virtual method not found: " + ArtMethod* method = klass->FindClassMethod(method_name, signature, pointer_size); + CHECK(method != nullptr && !method->IsDirect()) << "Virtual method not found: " << class_name << "." << method_name << signature; CompileMethod(method); } diff --git a/compiler/compiled_method.h b/compiler/compiled_method.h index 0ca23a5c50..761e9e19a8 100644 --- a/compiler/compiled_method.h +++ b/compiler/compiled_method.h @@ -120,13 +120,13 @@ class LinkerPatch { // patch_type_ as an uintN_t and do explicit static_cast<>s. enum class Type : uint8_t { kMethodRelative, // NOTE: Actual patching is instruction_set-dependent. + kMethodBssEntry, // NOTE: Actual patching is instruction_set-dependent. kCall, kCallRelative, // NOTE: Actual patching is instruction_set-dependent. kTypeRelative, // NOTE: Actual patching is instruction_set-dependent. kTypeBssEntry, // NOTE: Actual patching is instruction_set-dependent. kStringRelative, // NOTE: Actual patching is instruction_set-dependent. kStringBssEntry, // NOTE: Actual patching is instruction_set-dependent. - kDexCacheArray, // NOTE: Actual patching is instruction_set-dependent. kBakerReadBarrierBranch, // NOTE: Actual patching is instruction_set-dependent. }; @@ -140,6 +140,16 @@ class LinkerPatch { return patch; } + static LinkerPatch MethodBssEntryPatch(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t target_method_idx) { + LinkerPatch patch(literal_offset, Type::kMethodBssEntry, target_dex_file); + patch.method_idx_ = target_method_idx; + patch.pc_insn_offset_ = pc_insn_offset; + return patch; + } + static LinkerPatch CodePatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t target_method_idx) { @@ -196,16 +206,6 @@ class LinkerPatch { return patch; } - static LinkerPatch DexCacheArrayPatch(size_t literal_offset, - const DexFile* target_dex_file, - uint32_t pc_insn_offset, - uint32_t element_offset) { - LinkerPatch patch(literal_offset, Type::kDexCacheArray, target_dex_file); - patch.pc_insn_offset_ = pc_insn_offset; - patch.element_offset_ = element_offset; - return patch; - } - static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset, uint32_t custom_value1 = 0u, uint32_t custom_value2 = 0u) { @@ -229,12 +229,12 @@ class LinkerPatch { bool IsPcRelative() const { switch (GetType()) { case Type::kMethodRelative: + case Type::kMethodBssEntry: case Type::kCallRelative: case Type::kTypeRelative: case Type::kTypeBssEntry: case Type::kStringRelative: case Type::kStringBssEntry: - case Type::kDexCacheArray: case Type::kBakerReadBarrierBranch: return true; default: @@ -244,6 +244,7 @@ class LinkerPatch { MethodReference TargetMethod() const { DCHECK(patch_type_ == Type::kMethodRelative || + patch_type_ == Type::kMethodBssEntry || patch_type_ == Type::kCall || patch_type_ == Type::kCallRelative); return MethodReference(target_dex_file_, method_idx_); @@ -273,23 +274,13 @@ class LinkerPatch { return dex::StringIndex(string_idx_); } - const DexFile* TargetDexCacheDexFile() const { - DCHECK(patch_type_ == Type::kDexCacheArray); - return target_dex_file_; - } - - size_t TargetDexCacheElementOffset() const { - DCHECK(patch_type_ == Type::kDexCacheArray); - return element_offset_; - } - uint32_t PcInsnOffset() const { DCHECK(patch_type_ == Type::kMethodRelative || + patch_type_ == Type::kMethodBssEntry || patch_type_ == Type::kTypeRelative || patch_type_ == Type::kTypeBssEntry || patch_type_ == Type::kStringRelative || - patch_type_ == Type::kStringBssEntry || - patch_type_ == Type::kDexCacheArray); + patch_type_ == Type::kStringBssEntry); return pc_insn_offset_; } @@ -324,12 +315,10 @@ class LinkerPatch { uint32_t method_idx_; // Method index for Call/Method patches. uint32_t type_idx_; // Type index for Type patches. uint32_t string_idx_; // String index for String patches. - uint32_t element_offset_; // Element offset in the dex cache arrays. uint32_t baker_custom_value1_; static_assert(sizeof(method_idx_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(type_idx_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(string_idx_) == sizeof(cmp1_), "needed by relational operators"); - static_assert(sizeof(element_offset_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(baker_custom_value1_) == sizeof(cmp1_), "needed by relational operators"); }; union { diff --git a/compiler/compiled_method_test.cc b/compiler/compiled_method_test.cc index 72b2282ade..f4a72cf2cc 100644 --- a/compiler/compiled_method_test.cc +++ b/compiler/compiled_method_test.cc @@ -58,6 +58,14 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::RelativeMethodPatch(16u, dex_file2, 3001u, 1000u), LinkerPatch::RelativeMethodPatch(16u, dex_file2, 3000u, 1001u), LinkerPatch::RelativeMethodPatch(16u, dex_file2, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3001u, 1001u), LinkerPatch::CodePatch(16u, dex_file1, 1000u), LinkerPatch::CodePatch(16u, dex_file1, 1001u), LinkerPatch::CodePatch(16u, dex_file2, 1000u), @@ -98,14 +106,6 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::StringBssEntryPatch(16u, dex_file2, 3001u, 1000u), LinkerPatch::StringBssEntryPatch(16u, dex_file2, 3000u, 1001u), LinkerPatch::StringBssEntryPatch(16u, dex_file2, 3001u, 1001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3001u, 2001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3001u, 2001u), LinkerPatch::BakerReadBarrierBranchPatch(16u, 0u, 0u), LinkerPatch::BakerReadBarrierBranchPatch(16u, 0u, 1u), LinkerPatch::BakerReadBarrierBranchPatch(16u, 1u, 0u), @@ -119,6 +119,14 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::RelativeMethodPatch(32u, dex_file2, 3001u, 1000u), LinkerPatch::RelativeMethodPatch(32u, dex_file2, 3000u, 1001u), LinkerPatch::RelativeMethodPatch(32u, dex_file2, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3001u, 1001u), LinkerPatch::CodePatch(32u, dex_file1, 1000u), LinkerPatch::CodePatch(32u, dex_file1, 1001u), LinkerPatch::CodePatch(32u, dex_file2, 1000u), @@ -159,14 +167,6 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::StringBssEntryPatch(32u, dex_file2, 3001u, 1000u), LinkerPatch::StringBssEntryPatch(32u, dex_file2, 3000u, 1001u), LinkerPatch::StringBssEntryPatch(32u, dex_file2, 3001u, 1001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3001u, 2001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3001u, 2001u), LinkerPatch::BakerReadBarrierBranchPatch(32u, 0u, 0u), LinkerPatch::BakerReadBarrierBranchPatch(32u, 0u, 1u), LinkerPatch::BakerReadBarrierBranchPatch(32u, 1u, 0u), diff --git a/compiler/compiler.h b/compiler/compiler.h index cd4c59101e..ba89cb1aed 100644 --- a/compiler/compiler.h +++ b/compiler/compiler.h @@ -25,6 +25,7 @@ namespace art { namespace jit { class JitCodeCache; + class JitLogger; } // namespace jit namespace mirror { class ClassLoader; @@ -76,7 +77,8 @@ class Compiler { virtual bool JitCompile(Thread* self ATTRIBUTE_UNUSED, jit::JitCodeCache* code_cache ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED, - bool osr ATTRIBUTE_UNUSED) + bool osr ATTRIBUTE_UNUSED, + jit::JitLogger* jit_logger ATTRIBUTE_UNUSED) REQUIRES_SHARED(Locks::mutator_lock_) { return false; } diff --git a/compiler/dex/dex_to_dex_compiler.cc b/compiler/dex/dex_to_dex_compiler.cc index 2db99cda3e..9d57b965ab 100644 --- a/compiler/dex/dex_to_dex_compiler.cc +++ b/compiler/dex/dex_to_dex_compiler.cc @@ -22,12 +22,14 @@ #include "art_method-inl.h" #include "base/logging.h" #include "base/mutex.h" +#include "bytecode_utils.h" #include "compiled_method.h" #include "dex_file-inl.h" #include "dex_instruction-inl.h" #include "driver/compiler_driver.h" #include "driver/dex_compilation_unit.h" #include "mirror/dex_cache.h" +#include "quicken_info.h" #include "thread-current-inl.h" namespace art { @@ -110,13 +112,9 @@ class DexCompiler { void DexCompiler::Compile() { DCHECK_EQ(dex_to_dex_compilation_level_, DexToDexCompilationLevel::kOptimize); - const DexFile::CodeItem* code_item = unit_.GetCodeItem(); - const uint16_t* insns = code_item->insns_; - const uint32_t insns_size = code_item->insns_size_in_code_units_; - Instruction* inst = const_cast<Instruction*>(Instruction::At(insns)); - - for (uint32_t dex_pc = 0; dex_pc < insns_size; - inst = const_cast<Instruction*>(inst->Next()), dex_pc = inst->GetDexPc(insns)) { + for (CodeItemIterator it(*unit_.GetCodeItem()); !it.Done(); it.Advance()) { + Instruction* inst = const_cast<Instruction*>(&it.CurrentInstruction()); + const uint32_t dex_pc = it.CurrentDexPc(); switch (inst->Opcode()) { case Instruction::RETURN_VOID: CompileReturnVoid(inst, dex_pc); @@ -124,6 +122,11 @@ void DexCompiler::Compile() { case Instruction::CHECK_CAST: inst = CompileCheckCast(inst, dex_pc); + if (inst->Opcode() == Instruction::NOP) { + // We turned the CHECK_CAST into two NOPs, avoid visiting the second NOP twice since this + // would add 2 quickening info entries. + it.Advance(); + } break; case Instruction::IGET: @@ -190,7 +193,14 @@ void DexCompiler::Compile() { CompileInvokeVirtual(inst, dex_pc, Instruction::INVOKE_VIRTUAL_RANGE_QUICK, true); break; + case Instruction::NOP: + // We need to differentiate between check cast inserted NOP and normal NOP, put an invalid + // index in the map for normal nops. This should be rare in real code. + quickened_info_.push_back(QuickenedInfo(dex_pc, DexFile::kDexNoIndex16)); + break; + default: + DCHECK(!inst->IsQuickened()); // Nothing to do. break; } @@ -281,13 +291,14 @@ void DexCompiler::CompileInvokeVirtual(Instruction* inst, uint32_t dex_pc, ScopedObjectAccess soa(Thread::Current()); ClassLinker* class_linker = unit_.GetClassLinker(); - ArtMethod* resolved_method = class_linker->ResolveMethod<ClassLinker::kForceICCECheck>( - GetDexFile(), - method_idx, - unit_.GetDexCache(), - unit_.GetClassLoader(), - /* referrer */ nullptr, - kVirtual); + ArtMethod* resolved_method = + class_linker->ResolveMethod<ClassLinker::ResolveMode::kCheckICCEAndIAE>( + GetDexFile(), + method_idx, + unit_.GetDexCache(), + unit_.GetClassLoader(), + /* referrer */ nullptr, + kVirtual); if (UNLIKELY(resolved_method == nullptr)) { // Clean up any exception left by type resolution. @@ -348,10 +359,26 @@ CompiledMethod* ArtCompileDEX( } // Create a `CompiledMethod`, with the quickened information in the vmap table. - Leb128EncodingVector<> builder; + if (kIsDebugBuild) { + // Double check that the counts line up with the size of the quicken info. + size_t quicken_count = 0; + for (CodeItemIterator it(*code_item); !it.Done(); it.Advance()) { + if (QuickenInfoTable::NeedsIndexForInstruction(&it.CurrentInstruction())) { + ++quicken_count; + } + } + CHECK_EQ(quicken_count, dex_compiler.GetQuickenedInfo().size()); + } + std::vector<uint8_t> quicken_data; for (QuickenedInfo info : dex_compiler.GetQuickenedInfo()) { - builder.PushBackUnsigned(info.dex_pc); - builder.PushBackUnsigned(info.dex_member_index); + // Dex pc is not serialized, only used for checking the instructions. Since we access the + // array based on the index of the quickened instruction, the indexes must line up perfectly. + // The reader side uses the NeedsIndexForInstruction function too. + const Instruction* inst = Instruction::At(code_item->insns_ + info.dex_pc); + CHECK(QuickenInfoTable::NeedsIndexForInstruction(inst)) << inst->Opcode(); + // Add the index. + quicken_data.push_back(static_cast<uint8_t>(info.dex_member_index >> 0)); + quicken_data.push_back(static_cast<uint8_t>(info.dex_member_index >> 8)); } InstructionSet instruction_set = driver->GetInstructionSet(); if (instruction_set == kThumb2) { @@ -366,7 +393,7 @@ CompiledMethod* ArtCompileDEX( 0, 0, ArrayRef<const uint8_t>(), // method_info - ArrayRef<const uint8_t>(builder.GetData()), // vmap_table + ArrayRef<const uint8_t>(quicken_data), // vmap_table ArrayRef<const uint8_t>(), // cfi data ArrayRef<const LinkerPatch>()); } diff --git a/compiler/dex/dex_to_dex_decompiler_test.cc b/compiler/dex/dex_to_dex_decompiler_test.cc index e486e2e6ec..8e416b0480 100644 --- a/compiler/dex/dex_to_dex_decompiler_test.cc +++ b/compiler/dex/dex_to_dex_decompiler_test.cc @@ -27,8 +27,10 @@ #include "verifier/method_verifier-inl.h" #include "mirror/class_loader.h" #include "runtime.h" -#include "thread.h" #include "scoped_thread_state_change-inl.h" +#include "thread.h" +#include "verifier/method_verifier-inl.h" +#include "verifier/verifier_deps.h" namespace art { @@ -39,10 +41,12 @@ class DexToDexDecompilerTest : public CommonCompilerTest { TimingLogger::ScopedTiming t(__FUNCTION__, &timings); compiler_options_->boot_image_ = false; compiler_options_->SetCompilerFilter(CompilerFilter::kQuicken); - compiler_driver_->CompileAll(class_loader, - GetDexFiles(class_loader), - /* verifier_deps */ nullptr, - &timings); + // Create the main VerifierDeps, here instead of in the compiler since we want to aggregate + // the results for all the dex files, not just the results for the current dex file. + Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps( + new verifier::VerifierDeps(GetDexFiles(class_loader))); + compiler_driver_->SetDexFilesForOatFile(GetDexFiles(class_loader)); + compiler_driver_->CompileAll(class_loader, GetDexFiles(class_loader), &timings); } void RunTest(const char* dex_name) { @@ -85,13 +89,7 @@ class DexToDexDecompilerTest : public CommonCompilerTest { continue; } ClassDataItemIterator it(*updated_dex_file, class_data); - // Skip fields - while (it.HasNextStaticField()) { - it.Next(); - } - while (it.HasNextInstanceField()) { - it.Next(); - } + it.SkipAllFields(); // Unquicken each method. while (it.HasNextDirectMethod()) { diff --git a/compiler/dex/inline_method_analyser.cc b/compiler/dex/inline_method_analyser.cc index 257229101c..e5ff7fcace 100644 --- a/compiler/dex/inline_method_analyser.cc +++ b/compiler/dex/inline_method_analyser.cc @@ -145,9 +145,8 @@ ArtMethod* GetTargetConstructor(ArtMethod* method, const Instruction* invoke_dir DCHECK_EQ(invoke_direct->VRegC_35c(), method->GetCodeItem()->registers_size_ - method->GetCodeItem()->ins_size_); uint32_t method_index = invoke_direct->VRegB_35c(); - PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize(); - ArtMethod* target_method = - method->GetDexCache()->GetResolvedMethod(method_index, pointer_size); + ArtMethod* target_method = Runtime::Current()->GetClassLinker()->LookupResolvedMethod( + method_index, method->GetDexCache(), method->GetClassLoader()); if (kIsDebugBuild && target_method != nullptr) { CHECK(!target_method->IsStatic()); CHECK(target_method->IsConstructor()); diff --git a/compiler/dex/quick_compiler_callbacks.cc b/compiler/dex/quick_compiler_callbacks.cc index 932eb51aee..1a240bdc4c 100644 --- a/compiler/dex/quick_compiler_callbacks.cc +++ b/compiler/dex/quick_compiler_callbacks.cc @@ -16,17 +16,35 @@ #include "quick_compiler_callbacks.h" +#include "driver/compiler_driver.h" #include "verifier/method_verifier-inl.h" #include "verification_results.h" namespace art { void QuickCompilerCallbacks::MethodVerified(verifier::MethodVerifier* verifier) { - verification_results_->ProcessVerifiedMethod(verifier); + if (verification_results_ != nullptr) { + verification_results_->ProcessVerifiedMethod(verifier); + } } void QuickCompilerCallbacks::ClassRejected(ClassReference ref) { - verification_results_->AddRejectedClass(ref); + if (verification_results_ != nullptr) { + verification_results_->AddRejectedClass(ref); + } +} + +bool QuickCompilerCallbacks::CanAssumeVerified(ClassReference ref) { + // If we don't have class unloading enabled in the compiler, we will never see class that were + // previously verified. Return false to avoid overhead from the lookup in the compiler driver. + if (!does_class_unloading_) { + return false; + } + DCHECK(compiler_driver_ != nullptr); + // In the case of the quicken filter: avoiding verification of quickened instructions, which the + // verifier doesn't currently support. + // In the case of the verify filter, avoiding verifiying twice. + return compiler_driver_->CanAssumeVerified(ref); } } // namespace art diff --git a/compiler/dex/quick_compiler_callbacks.h b/compiler/dex/quick_compiler_callbacks.h index db0fdaa72f..578aff45e5 100644 --- a/compiler/dex/quick_compiler_callbacks.h +++ b/compiler/dex/quick_compiler_callbacks.h @@ -22,17 +22,13 @@ namespace art { +class CompilerDriver; class VerificationResults; class QuickCompilerCallbacks FINAL : public CompilerCallbacks { public: - QuickCompilerCallbacks(VerificationResults* verification_results, - CompilerCallbacks::CallbackMode mode) - : CompilerCallbacks(mode), - verification_results_(verification_results), - verifier_deps_(nullptr) { - CHECK(verification_results != nullptr); - } + explicit QuickCompilerCallbacks(CompilerCallbacks::CallbackMode mode) + : CompilerCallbacks(mode) {} ~QuickCompilerCallbacks() { } @@ -54,8 +50,23 @@ class QuickCompilerCallbacks FINAL : public CompilerCallbacks { verifier_deps_.reset(deps); } + void SetVerificationResults(VerificationResults* verification_results) { + verification_results_ = verification_results; + } + + bool CanAssumeVerified(ClassReference ref) OVERRIDE; + + void SetDoesClassUnloading(bool does_class_unloading, CompilerDriver* compiler_driver) + OVERRIDE { + does_class_unloading_ = does_class_unloading; + compiler_driver_ = compiler_driver; + DCHECK(!does_class_unloading || compiler_driver_ != nullptr); + } + private: - VerificationResults* const verification_results_; + VerificationResults* verification_results_ = nullptr; + bool does_class_unloading_ = false; + CompilerDriver* compiler_driver_ = nullptr; std::unique_ptr<verifier::VerifierDeps> verifier_deps_; }; diff --git a/compiler/dex/verification_results.cc b/compiler/dex/verification_results.cc index b87cb61ed6..beb3439e62 100644 --- a/compiler/dex/verification_results.cc +++ b/compiler/dex/verification_results.cc @@ -24,7 +24,7 @@ #include "runtime.h" #include "thread.h" #include "thread-current-inl.h" -#include "utils/atomic_method_ref_map-inl.h" +#include "utils/atomic_dex_ref_map-inl.h" #include "verified_method.h" #include "verifier/method_verifier-inl.h" @@ -38,7 +38,7 @@ VerificationResults::VerificationResults(const CompilerOptions* compiler_options VerificationResults::~VerificationResults() { WriterMutexLock mu(Thread::Current(), verified_methods_lock_); STLDeleteValues(&verified_methods_); - atomic_verified_methods_.Visit([](const MethodReference& ref ATTRIBUTE_UNUSED, + atomic_verified_methods_.Visit([](const DexFileReference& ref ATTRIBUTE_UNUSED, const VerifiedMethod* method) { delete method; }); @@ -46,22 +46,28 @@ VerificationResults::~VerificationResults() { void VerificationResults::ProcessVerifiedMethod(verifier::MethodVerifier* method_verifier) { DCHECK(method_verifier != nullptr); + if (!compiler_options_->IsAnyCompilationEnabled()) { + // Verified methods are only required for quickening and compilation. + return; + } MethodReference ref = method_verifier->GetMethodReference(); std::unique_ptr<const VerifiedMethod> verified_method(VerifiedMethod::Create(method_verifier)); if (verified_method == nullptr) { // We'll punt this later. return; } - AtomicMap::InsertResult result = atomic_verified_methods_.Insert(ref, - /*expected*/ nullptr, - verified_method.get()); + AtomicMap::InsertResult result = atomic_verified_methods_.Insert( + DexFileReference(ref.dex_file, ref.dex_method_index), + /*expected*/ nullptr, + verified_method.get()); const VerifiedMethod* existing = nullptr; bool inserted; if (result != AtomicMap::kInsertResultInvalidDexFile) { inserted = (result == AtomicMap::kInsertResultSuccess); if (!inserted) { // Rare case. - CHECK(atomic_verified_methods_.Get(ref, &existing)); + CHECK(atomic_verified_methods_.Get(DexFileReference(ref.dex_file, ref.dex_method_index), + &existing)); CHECK_NE(verified_method.get(), existing); } } else { @@ -98,7 +104,8 @@ void VerificationResults::ProcessVerifiedMethod(verifier::MethodVerifier* method const VerifiedMethod* VerificationResults::GetVerifiedMethod(MethodReference ref) { const VerifiedMethod* ret = nullptr; - if (atomic_verified_methods_.Get(ref, &ret)) { + DCHECK(compiler_options_->IsAnyCompilationEnabled()); + if (atomic_verified_methods_.Get(DexFileReference(ref.dex_file, ref.dex_method_index), &ret)) { return ret; } ReaderMutexLock mu(Thread::Current(), verified_methods_lock_); @@ -110,12 +117,14 @@ void VerificationResults::CreateVerifiedMethodFor(MethodReference ref) { // This method should only be called for classes verified at compile time, // which have no verifier error, nor has methods that we know will throw // at runtime. - atomic_verified_methods_.Insert( - ref, - /*expected*/ nullptr, - new VerifiedMethod(/* encountered_error_types */ 0, /* has_runtime_throw */ false)); - // We don't check the result of `Insert` as we could insert twice for the same - // MethodReference in the presence of duplicate methods. + std::unique_ptr<VerifiedMethod> verified_method = std::make_unique<VerifiedMethod>( + /* encountered_error_types */ 0, /* has_runtime_throw */ false); + if (atomic_verified_methods_.Insert(DexFileReference(ref.dex_file, ref.dex_method_index), + /*expected*/ nullptr, + verified_method.get()) == + AtomicMap::InsertResult::kInsertResultSuccess) { + verified_method.release(); + } } void VerificationResults::AddRejectedClass(ClassReference ref) { @@ -145,7 +154,7 @@ bool VerificationResults::IsCandidateForCompilation(MethodReference&, } void VerificationResults::AddDexFile(const DexFile* dex_file) { - atomic_verified_methods_.AddDexFile(dex_file); + atomic_verified_methods_.AddDexFile(dex_file, dex_file->NumMethodIds()); WriterMutexLock mu(Thread::Current(), verified_methods_lock_); // There can be some verified methods that are already registered for the dex_file since we set // up well known classes earlier. Remove these and put them in the array so that we don't @@ -153,7 +162,9 @@ void VerificationResults::AddDexFile(const DexFile* dex_file) { for (auto it = verified_methods_.begin(); it != verified_methods_.end(); ) { MethodReference ref = it->first; if (ref.dex_file == dex_file) { - CHECK(atomic_verified_methods_.Insert(ref, nullptr, it->second) == + CHECK(atomic_verified_methods_.Insert(DexFileReference(ref.dex_file, ref.dex_method_index), + nullptr, + it->second) == AtomicMap::kInsertResultSuccess); it = verified_methods_.erase(it); } else { diff --git a/compiler/dex/verification_results.h b/compiler/dex/verification_results.h index 22749fa621..5a03599de0 100644 --- a/compiler/dex/verification_results.h +++ b/compiler/dex/verification_results.h @@ -26,7 +26,7 @@ #include "class_reference.h" #include "method_reference.h" #include "safe_map.h" -#include "utils/atomic_method_ref_map.h" +#include "utils/atomic_dex_ref_map.h" namespace art { @@ -64,7 +64,7 @@ class VerificationResults { private: // Verified methods. The method array is fixed to avoid needing a lock to extend it. - using AtomicMap = AtomicMethodRefMap<const VerifiedMethod*>; + using AtomicMap = AtomicDexRefMap<const VerifiedMethod*>; using VerifiedMethodMap = SafeMap<MethodReference, const VerifiedMethod*, MethodReferenceComparator>; diff --git a/compiler/driver/compiler_driver-inl.h b/compiler/driver/compiler_driver-inl.h index 8cc1cc38e2..b04392918d 100644 --- a/compiler/driver/compiler_driver-inl.h +++ b/compiler/driver/compiler_driver-inl.h @@ -102,46 +102,17 @@ inline std::pair<bool, bool> CompilerDriver::IsFastInstanceField( return std::make_pair(fast_get, fast_put); } -template <typename ArtMember> -inline bool CompilerDriver::CanAccessResolvedMember(mirror::Class* referrer_class ATTRIBUTE_UNUSED, - mirror::Class* access_to ATTRIBUTE_UNUSED, - ArtMember* member ATTRIBUTE_UNUSED, - mirror::DexCache* dex_cache ATTRIBUTE_UNUSED, - uint32_t field_idx ATTRIBUTE_UNUSED) { - // Not defined for ArtMember values other than ArtField or ArtMethod. - UNREACHABLE(); -} - -template <> -inline bool CompilerDriver::CanAccessResolvedMember<ArtField>(mirror::Class* referrer_class, - mirror::Class* access_to, - ArtField* field, - mirror::DexCache* dex_cache, - uint32_t field_idx) { - return referrer_class->CanAccessResolvedField(access_to, field, dex_cache, field_idx); -} - -template <> -inline bool CompilerDriver::CanAccessResolvedMember<ArtMethod>( - mirror::Class* referrer_class, - mirror::Class* access_to, - ArtMethod* method, - mirror::DexCache* dex_cache, - uint32_t field_idx) { - return referrer_class->CanAccessResolvedMethod(access_to, method, dex_cache, field_idx); -} - inline ArtMethod* CompilerDriver::ResolveMethod( - ScopedObjectAccess& soa, Handle<mirror::DexCache> dex_cache, - Handle<mirror::ClassLoader> class_loader, const DexCompilationUnit* mUnit, - uint32_t method_idx, InvokeType invoke_type, bool check_incompatible_class_change) { + ScopedObjectAccess& soa, + Handle<mirror::DexCache> dex_cache, + Handle<mirror::ClassLoader> class_loader, + const DexCompilationUnit* mUnit, + uint32_t method_idx, + InvokeType invoke_type) { DCHECK_EQ(class_loader.Get(), mUnit->GetClassLoader().Get()); ArtMethod* resolved_method = - check_incompatible_class_change - ? mUnit->GetClassLinker()->ResolveMethod<ClassLinker::kForceICCECheck>( - *dex_cache->GetDexFile(), method_idx, dex_cache, class_loader, nullptr, invoke_type) - : mUnit->GetClassLinker()->ResolveMethod<ClassLinker::kNoICCECheckForCache>( - *dex_cache->GetDexFile(), method_idx, dex_cache, class_loader, nullptr, invoke_type); + mUnit->GetClassLinker()->ResolveMethod<ClassLinker::ResolveMode::kCheckICCEAndIAE>( + *dex_cache->GetDexFile(), method_idx, dex_cache, class_loader, nullptr, invoke_type); if (UNLIKELY(resolved_method == nullptr)) { DCHECK(soa.Self()->IsExceptionPending()); // Clean up any exception left by type resolution. diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 93f678c64a..e48b82d264 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -41,42 +41,42 @@ #include "compiler.h" #include "compiler_callbacks.h" #include "compiler_driver-inl.h" -#include "dex_compilation_unit.h" -#include "dex_file-inl.h" -#include "dex_instruction-inl.h" #include "dex/dex_to_dex_compiler.h" #include "dex/verification_results.h" #include "dex/verified_method.h" +#include "dex_compilation_unit.h" +#include "dex_file-inl.h" +#include "dex_instruction-inl.h" #include "driver/compiler_options.h" -#include "intrinsics_enum.h" -#include "jni_internal.h" -#include "object_lock.h" -#include "runtime.h" #include "gc/accounting/card_table-inl.h" #include "gc/accounting/heap_bitmap.h" #include "gc/space/image_space.h" #include "gc/space/space.h" -#include "mirror/class_loader.h" +#include "handle_scope-inl.h" +#include "intrinsics_enum.h" +#include "jni_internal.h" #include "mirror/class-inl.h" +#include "mirror/class_loader.h" #include "mirror/dex_cache-inl.h" #include "mirror/object-inl.h" #include "mirror/object-refvisitor-inl.h" #include "mirror/object_array-inl.h" #include "mirror/throwable.h" +#include "nativehelper/ScopedLocalRef.h" +#include "object_lock.h" +#include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "ScopedLocalRef.h" -#include "handle_scope-inl.h" #include "thread.h" #include "thread_list.h" #include "thread_pool.h" #include "trampolines/trampoline_compiler.h" #include "transaction.h" -#include "utils/atomic_method_ref_map-inl.h" +#include "utils/atomic_dex_ref_map-inl.h" #include "utils/dex_cache_arrays_layout-inl.h" #include "utils/swap_space.h" #include "vdex_file.h" -#include "verifier/method_verifier.h" #include "verifier/method_verifier-inl.h" +#include "verifier/method_verifier.h" #include "verifier/verifier_deps.h" #include "verifier/verifier_enums.h" @@ -87,6 +87,10 @@ static constexpr bool kTimeCompileMethod = !kIsDebugBuild; // Print additional info during profile guided compilation. static constexpr bool kDebugProfileGuidedCompilation = false; +// Max encoded fields allowed for initializing app image. Hardcode the number for now +// because 5000 should be large enough. +static constexpr uint32_t kMaxEncodedFields = 5000; + static double Percentage(size_t x, size_t y) { return 100.0 * (static_cast<double>(x)) / (static_cast<double>(x + y)); } @@ -287,7 +291,6 @@ CompilerDriver::CompilerDriver( instruction_set_(instruction_set == kArm ? kThumb2 : instruction_set), instruction_set_features_(instruction_set_features), requires_constructor_barrier_lock_("constructor barrier lock"), - compiled_classes_lock_("compiled classes lock"), non_relative_linker_patch_count_(0u), image_classes_(image_classes), classes_to_compile_(compiled_classes), @@ -300,7 +303,6 @@ CompilerDriver::CompilerDriver( timings_logger_(timer), compiler_context_(nullptr), support_boot_image_fixup_(true), - dex_files_for_oat_file_(nullptr), compiled_method_storage_(swap_fd), profile_compilation_info_(profile_compilation_info), max_arena_alloc_(0), @@ -317,7 +319,7 @@ CompilerDriver::CompilerDriver( } CompilerDriver::~CompilerDriver() { - compiled_methods_.Visit([this](const MethodReference& ref ATTRIBUTE_UNUSED, + compiled_methods_.Visit([this](const DexFileReference& ref ATTRIBUTE_UNUSED, CompiledMethod* method) { if (method != nullptr) { CompiledMethod::ReleaseSwapAllocatedCompiledMethod(this, method); @@ -370,14 +372,12 @@ static void SetupIntrinsic(Thread* self, REQUIRES_SHARED(Locks::mutator_lock_) { ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); PointerSize image_size = class_linker->GetImagePointerSize(); - mirror::Class* cls = class_linker->FindSystemClass(self, class_name); + ObjPtr<mirror::Class> cls = class_linker->FindSystemClass(self, class_name); if (cls == nullptr) { LOG(FATAL) << "Could not find class of intrinsic " << class_name; } - ArtMethod* method = (invoke_type == kStatic || invoke_type == kDirect) - ? cls->FindDeclaredDirectMethod(method_name, signature, image_size) - : cls->FindDeclaredVirtualMethod(method_name, signature, image_size); - if (method == nullptr) { + ArtMethod* method = cls->FindClassMethod(method_name, signature, image_size); + if (method == nullptr || method->GetDeclaringClass() != cls) { LOG(FATAL) << "Could not find method of intrinsic " << class_name << " " << method_name << " " << signature; } @@ -425,26 +425,6 @@ INTRINSICS_LIST(SETUP_INTRINSICS) FreeThreadPools(); } -void CompilerDriver::CompileAll(jobject class_loader, - const std::vector<const DexFile*>& dex_files, - VdexFile* vdex_file, - TimingLogger* timings) { - if (vdex_file != nullptr) { - // TODO: we unquicken unconditionnally, as we don't know - // if the boot image has changed. How exactly we'll know is under - // experimentation. - TimingLogger::ScopedTiming t("Unquicken", timings); - // We do not decompile a RETURN_VOID_NO_BARRIER into a RETURN_VOID, as the quickening - // optimization does not depend on the boot image (the optimization relies on not - // having final fields in a class, which does not change for an app). - VdexFile::Unquicken(dex_files, vdex_file->GetQuickeningInfo()); - - Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps( - new verifier::VerifierDeps(dex_files, vdex_file->GetVerifierDepsData())); - } - CompileAll(class_loader, dex_files, timings); -} - static optimizer::DexToDexCompilationLevel GetDexToDexCompilationLevel( Thread* self, const CompilerDriver& driver, Handle<mirror::ClassLoader> class_loader, const DexFile& dex_file, const DexFile::ClassDef& class_def) @@ -530,8 +510,9 @@ static void CompileMethod(Thread* self, // TODO: Refactor the compilation to avoid having to distinguish the two passes // here. That should be done on a higher level. http://b/29089975 if (driver->GetCurrentDexToDexMethods()->IsBitSet(method_idx)) { - const VerifiedMethod* verified_method = - driver->GetVerificationResults()->GetVerifiedMethod(method_ref); + VerificationResults* results = driver->GetVerificationResults(); + DCHECK(results != nullptr); + const VerifiedMethod* verified_method = results->GetVerifiedMethod(method_ref); // Do not optimize if a VerifiedMethod is missing. SafeCast elision, // for example, relies on it. compiled_method = optimizer::ArtCompileDEX( @@ -559,7 +540,7 @@ static void CompileMethod(Thread* self, // TODO: Lookup annotation from DexFile directly without resolving method. ArtMethod* method = - Runtime::Current()->GetClassLinker()->ResolveMethod<ClassLinker::kNoICCECheckForCache>( + Runtime::Current()->GetClassLinker()->ResolveMethod<ClassLinker::ResolveMode::kNoChecks>( dex_file, method_idx, dex_cache, @@ -592,12 +573,12 @@ static void CompileMethod(Thread* self, } else if ((access_flags & kAccAbstract) != 0) { // Abstract methods don't have code. } else { - const VerifiedMethod* verified_method = - driver->GetVerificationResults()->GetVerifiedMethod(method_ref); + VerificationResults* results = driver->GetVerificationResults(); + DCHECK(results != nullptr); + const VerifiedMethod* verified_method = results->GetVerifiedMethod(method_ref); bool compile = compilation_enabled && // Basic checks, e.g., not <clinit>. - driver->GetVerificationResults() - ->IsCandidateForCompilation(method_ref, access_flags) && + results->IsCandidateForCompilation(method_ref, access_flags) && // Did not fail to create VerifiedMethod metadata. verified_method != nullptr && // Do not have failures that should punt to the interpreter. @@ -828,13 +809,7 @@ static void ResolveConstStrings(CompilerDriver* driver, } ClassDataItemIterator it(*dex_file, class_data); - // Skip fields - while (it.HasNextStaticField()) { - it.Next(); - } - while (it.HasNextInstanceField()) { - it.Next(); - } + it.SkipAllFields(); bool compilation_enabled = driver->IsClassToCompile( dex_file->StringByTypeIdx(class_def.class_idx_)); @@ -912,17 +887,18 @@ void CompilerDriver::PreCompile(jobject class_loader, TimingLogger* timings) { CheckThreadPools(); - for (const DexFile* dex_file : dex_files) { - // Can be already inserted if the caller is CompileOne. This happens for gtests. - if (!compiled_methods_.HaveDexFile(dex_file)) { - compiled_methods_.AddDexFile(dex_file); - } - } - LoadImageClasses(timings); VLOG(compiler) << "LoadImageClasses: " << GetMemoryUsageString(false); if (compiler_options_->IsAnyCompilationEnabled()) { + // Avoid adding the dex files in the case where we aren't going to add compiled methods. + // This reduces RAM usage for this case. + for (const DexFile* dex_file : dex_files) { + // Can be already inserted if the caller is CompileOne. This happens for gtests. + if (!compiled_methods_.HaveDexFile(dex_file)) { + compiled_methods_.AddDexFile(dex_file, dex_file->NumMethodIds()); + } + } // Resolve eagerly to prepare for compilation. Resolve(class_loader, dex_files, timings); VLOG(compiler) << "Resolve: " << GetMemoryUsageString(false); @@ -1000,8 +976,9 @@ bool CompilerDriver::ShouldCompileBasedOnProfile(const MethodReference& method_r if (profile_compilation_info_ == nullptr) { return false; } - // TODO: Revisit compiling all startup methods. b/36457259 - bool result = profile_compilation_info_->IsStartupOrHotMethod(method_ref); + // Compile only hot methods, it is the profile saver's job to decide what startup methods to mark + // as hot. + bool result = profile_compilation_info_->GetMethodHotness(method_ref).IsHot(); if (kDebugProfileGuidedCompilation) { LOG(INFO) << "[ProfileGuidedCompilation] " @@ -1680,9 +1657,7 @@ bool CompilerDriver::RequiresConstructorBarrier(const DexFile& dex_file, return false; } ClassDataItemIterator it(dex_file, class_data); - while (it.HasNextStaticField()) { - it.Next(); - } + it.SkipStaticFields(); // We require a constructor barrier if there are final instance fields. while (it.HasNextInstanceField()) { if (it.MemberIsFinal()) { @@ -1777,7 +1752,7 @@ class ResolveClassFieldsAndMethodsVisitor : public CompilationVisitor { } if (resolve_fields_and_methods) { while (it.HasNextDirectMethod()) { - ArtMethod* method = class_linker->ResolveMethod<ClassLinker::kNoICCECheckForCache>( + ArtMethod* method = class_linker->ResolveMethod<ClassLinker::ResolveMode::kNoChecks>( dex_file, it.GetMemberIndex(), dex_cache, class_loader, nullptr, it.GetMethodInvokeType(class_def)); if (method == nullptr) { @@ -1786,7 +1761,7 @@ class ResolveClassFieldsAndMethodsVisitor : public CompilationVisitor { it.Next(); } while (it.HasNextVirtualMethod()) { - ArtMethod* method = class_linker->ResolveMethod<ClassLinker::kNoICCECheckForCache>( + ArtMethod* method = class_linker->ResolveMethod<ClassLinker::ResolveMode::kNoChecks>( dex_file, it.GetMemberIndex(), dex_cache, class_loader, nullptr, it.GetMethodInvokeType(class_def)); if (method == nullptr) { @@ -1892,13 +1867,7 @@ static void PopulateVerifiedMethods(const DexFile& dex_file, return; } ClassDataItemIterator it(dex_file, class_data); - // Skip fields - while (it.HasNextStaticField()) { - it.Next(); - } - while (it.HasNextInstanceField()) { - it.Next(); - } + it.SkipAllFields(); while (it.HasNextDirectMethod()) { verification_results->CreateVerifiedMethodFor(MethodReference(&dex_file, it.GetMemberIndex())); @@ -1942,8 +1911,8 @@ bool CompilerDriver::FastVerify(jobject jclass_loader, TimingLogger* timings) { verifier::VerifierDeps* verifier_deps = Runtime::Current()->GetCompilerCallbacks()->GetVerifierDeps(); - // If there is an existing `VerifierDeps`, try to use it for fast verification. - if (verifier_deps == nullptr) { + // If there exist VerifierDeps that aren't the ones we just created to output, use them to verify. + if (verifier_deps == nullptr || verifier_deps->OutputOnly()) { return false; } TimingLogger::ScopedTiming t("Fast Verify", timings); @@ -1963,18 +1932,21 @@ bool CompilerDriver::FastVerify(jobject jclass_loader, // time. So instead we assume these classes still need to be verified at // runtime. for (const DexFile* dex_file : dex_files) { - // Fetch the list of unverified classes and turn it into a set for faster - // lookups. - const std::vector<dex::TypeIndex>& unverified_classes = + // Fetch the list of unverified classes. + const std::set<dex::TypeIndex>& unverified_classes = verifier_deps->GetUnverifiedClasses(*dex_file); - std::set<dex::TypeIndex> set(unverified_classes.begin(), unverified_classes.end()); for (uint32_t i = 0; i < dex_file->NumClassDefs(); ++i) { const DexFile::ClassDef& class_def = dex_file->GetClassDef(i); - if (set.find(class_def.class_idx_) == set.end()) { + if (unverified_classes.find(class_def.class_idx_) == unverified_classes.end()) { if (compiler_only_verifies) { // Just update the compiled_classes_ map. The compiler doesn't need to resolve // the type. - compiled_classes_.Overwrite(ClassReference(dex_file, i), mirror::Class::kStatusVerified); + DexFileReference ref(dex_file, i); + mirror::Class::Status existing = mirror::Class::kStatusNotReady; + DCHECK(compiled_classes_.Get(ref, &existing)) << ref.dex_file->GetLocation(); + ClassStateTable::InsertResult result = + compiled_classes_.Insert(ref, existing, mirror::Class::kStatusVerified); + CHECK_EQ(result, ClassStateTable::kInsertResultSuccess); } else { // Update the class status, so later compilation stages know they don't need to verify // the class. @@ -2014,14 +1986,16 @@ void CompilerDriver::Verify(jobject jclass_loader, // non boot image compilation. The verifier will need it to record the new dependencies. // Then dex2oat can update the vdex file with these new dependencies. if (!GetCompilerOptions().IsBootImage()) { + // Dex2oat creates the verifier deps. // Create the main VerifierDeps, and set it to this thread. - verifier::VerifierDeps* verifier_deps = new verifier::VerifierDeps(dex_files); - Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps(verifier_deps); + verifier::VerifierDeps* verifier_deps = + Runtime::Current()->GetCompilerCallbacks()->GetVerifierDeps(); + CHECK(verifier_deps != nullptr); Thread::Current()->SetVerifierDeps(verifier_deps); // Create per-thread VerifierDeps to avoid contention on the main one. // We will merge them after verification. for (ThreadPoolWorker* worker : parallel_thread_pool_->GetWorkers()) { - worker->GetThread()->SetVerifierDeps(new verifier::VerifierDeps(dex_files)); + worker->GetThread()->SetVerifierDeps(new verifier::VerifierDeps(dex_files_for_oat_file_)); } } @@ -2046,7 +2020,7 @@ void CompilerDriver::Verify(jobject jclass_loader, for (ThreadPoolWorker* worker : parallel_thread_pool_->GetWorkers()) { verifier::VerifierDeps* thread_deps = worker->GetThread()->GetVerifierDeps(); worker->GetThread()->SetVerifierDeps(nullptr); - verifier_deps->MergeWith(*thread_deps, dex_files);; + verifier_deps->MergeWith(*thread_deps, dex_files_for_oat_file_); delete thread_deps; } Thread::Current()->SetVerifierDeps(nullptr); @@ -2229,6 +2203,9 @@ void CompilerDriver::SetVerifiedDexFile(jobject class_loader, size_t thread_count, TimingLogger* timings) { TimingLogger::ScopedTiming t("Verify Dex File", timings); + if (!compiled_classes_.HaveDexFile(&dex_file)) { + compiled_classes_.AddDexFile(&dex_file, dex_file.NumClassDefs()); + } ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); ParallelCompilationManager context(class_linker, class_loader, this, &dex_file, dex_files, thread_pool); @@ -2271,8 +2248,15 @@ class InitializeClassVisitor : public CompilationVisitor { const char* descriptor = dex_file.StringDataByIdx(class_type_id.descriptor_idx_); ScopedObjectAccessUnchecked soa(Thread::Current()); StackHandleScope<3> hs(soa.Self()); + const bool is_boot_image = manager_->GetCompiler()->GetCompilerOptions().IsBootImage(); + const bool is_app_image = manager_->GetCompiler()->GetCompilerOptions().IsAppImage(); - mirror::Class::Status old_status = klass->GetStatus();; + mirror::Class::Status old_status = klass->GetStatus(); + // Don't initialize classes in boot space when compiling app image + if (is_app_image && klass->IsBootStrapClassLoaded()) { + // Also return early and don't store the class status in the recorded class status. + return; + } // Only try to initialize classes that were successfully verified. if (klass->IsVerified()) { // Attempt to initialize the class but bail if we either need to initialize the super-class @@ -2292,32 +2276,39 @@ class InitializeClassVisitor : public CompilationVisitor { ObjectLock<mirror::Class> lock(soa.Self(), h_klass); // Attempt to initialize allowing initialization of parent classes but still not static // fields. - bool is_superclass_initialized = true; - if (!manager_->GetCompiler()->GetCompilerOptions().IsAppImage()) { - // If not an app image case, the compiler won't initialize too much things and do a fast - // fail, don't check dependencies. + // Initialize dependencies first only for app image, to make TryInitialize recursive. + bool is_superclass_initialized = !is_app_image ? true : + InitializeDependencies(klass, class_loader, soa.Self()); + if (!is_app_image || (is_app_image && is_superclass_initialized)) { manager_->GetClassLinker()->EnsureInitialized(soa.Self(), klass, false, true); - } else { - // For app images, do the initialization recursively and resolve types encountered to make - // sure the compiler runs without error. - is_superclass_initialized = InitializeDependencies(klass, class_loader, soa.Self()); - if (is_superclass_initialized) { - manager_->GetClassLinker()->EnsureInitialized(soa.Self(), klass, false, true); - } } + // Otherwise it's in app image but superclasses can't be initialized, no need to proceed. old_status = klass->GetStatus(); - // If superclass cannot be initialized, no need to proceed. + + bool too_many_encoded_fields = false; + if (!is_boot_image && klass->NumStaticFields() > kMaxEncodedFields) { + too_many_encoded_fields = true; + } + // If the class was not initialized, we can proceed to see if we can initialize static + // fields. Limit the max number of encoded fields. if (!klass->IsInitialized() && + (is_app_image || is_boot_image) && is_superclass_initialized && + !too_many_encoded_fields && manager_->GetCompiler()->IsImageClass(descriptor)) { bool can_init_static_fields = false; - if (manager_->GetCompiler()->GetCompilerOptions().IsBootImage()) { + if (is_boot_image) { // We need to initialize static fields, we only do this for image classes that aren't - // marked with the $NoPreloadHolder (which implies this should not be initialized early). + // marked with the $NoPreloadHolder (which implies this should not be initialized + // early). can_init_static_fields = !StringPiece(descriptor).ends_with("$NoPreloadHolder;"); } else { - can_init_static_fields = manager_->GetCompiler()->GetCompilerOptions().IsAppImage() && + CHECK(is_app_image); + // The boot image case doesn't need to recursively initialize the dependencies with + // special logic since the class linker already does this. + can_init_static_fields = !soa.Self()->IsExceptionPending() && + is_superclass_initialized && NoClinitInDependency(klass, soa.Self(), &class_loader); // TODO The checking for clinit can be removed since it's already // checked when init superclass. Currently keep it because it contains @@ -2360,6 +2351,10 @@ class InitializeClassVisitor : public CompilationVisitor { soa.Self()->ClearException(); transaction.Rollback(); CHECK_EQ(old_status, klass->GetStatus()) << "Previous class status not restored"; + } else if (is_boot_image) { + // For boot image, we want to put the updated status in the oat class since we can't + // reject the image anyways. + old_status = klass->GetStatus(); } } @@ -2369,10 +2364,20 @@ class InitializeClassVisitor : public CompilationVisitor { // above as we will allocate strings, so must be allowed to suspend. if (&klass->GetDexFile() == manager_->GetDexFile()) { InternStrings(klass, class_loader); + } else { + DCHECK(!is_boot_image) << "Boot image must have equal dex files"; } } } } + // If the class still isn't initialized, at least try some checks that initialization + // would do so they can be skipped at runtime. + if (!klass->IsInitialized() && + manager_->GetClassLinker()->ValidateSuperClassDescriptors(klass)) { + old_status = mirror::Class::kStatusSuperclassValidated; + } else { + soa.Self()->ClearException(); + } soa.Self()->AssertNoPendingException(); } } @@ -2437,63 +2442,63 @@ class InitializeClassVisitor : public CompilationVisitor { bool ResolveTypesOfMethods(Thread* self, ArtMethod* m) REQUIRES_SHARED(Locks::mutator_lock_) { - auto rtn_type = m->GetReturnType(true); - if (rtn_type == nullptr) { - self->ClearException(); - return false; - } - const DexFile::TypeList* types = m->GetParameterTypeList(); - if (types != nullptr) { - for (uint32_t i = 0; i < types->Size(); ++i) { - dex::TypeIndex param_type_idx = types->GetTypeItem(i).type_idx_; - auto param_type = m->GetClassFromTypeIndex(param_type_idx, true); - if (param_type == nullptr) { - self->ClearException(); - return false; - } + auto rtn_type = m->GetReturnType(true); // return value is discarded because resolve will be done internally. + if (rtn_type == nullptr) { + self->ClearException(); + return false; + } + const DexFile::TypeList* types = m->GetParameterTypeList(); + if (types != nullptr) { + for (uint32_t i = 0; i < types->Size(); ++i) { + dex::TypeIndex param_type_idx = types->GetTypeItem(i).type_idx_; + auto param_type = m->GetClassFromTypeIndex(param_type_idx, true); + if (param_type == nullptr) { + self->ClearException(); + return false; } } - return true; + } + return true; } // Pre resolve types mentioned in all method signatures before start a transaction // since ResolveType doesn't work in transaction mode. bool PreResolveTypes(Thread* self, const Handle<mirror::Class>& klass) REQUIRES_SHARED(Locks::mutator_lock_) { - PointerSize pointer_size = manager_->GetClassLinker()->GetImagePointerSize(); - for (ArtMethod& m : klass->GetMethods(pointer_size)) { - if (!ResolveTypesOfMethods(self, &m)) { + PointerSize pointer_size = manager_->GetClassLinker()->GetImagePointerSize(); + for (ArtMethod& m : klass->GetMethods(pointer_size)) { + if (!ResolveTypesOfMethods(self, &m)) { + return false; + } + } + if (klass->IsInterface()) { + return true; + } else if (klass->HasSuperClass()) { + StackHandleScope<1> hs(self); + MutableHandle<mirror::Class> super_klass(hs.NewHandle<mirror::Class>(klass->GetSuperClass())); + for (int i = super_klass->GetVTableLength() - 1; i >= 0; --i) { + ArtMethod* m = klass->GetVTableEntry(i, pointer_size); + ArtMethod* super_m = super_klass->GetVTableEntry(i, pointer_size); + if (!ResolveTypesOfMethods(self, m) || !ResolveTypesOfMethods(self, super_m)) { return false; } } - if (klass->IsInterface()) { - return true; - } else if (klass->HasSuperClass()) { - StackHandleScope<1> hs(self); - MutableHandle<mirror::Class> super_klass(hs.NewHandle<mirror::Class>(klass->GetSuperClass())); - for (int i = super_klass->GetVTableLength() - 1; i >= 0; --i) { - ArtMethod* m = klass->GetVTableEntry(i, pointer_size); - ArtMethod* super_m = super_klass->GetVTableEntry(i, pointer_size); - if (!ResolveTypesOfMethods(self, m) || !ResolveTypesOfMethods(self, super_m)) { - return false; - } - } - for (int32_t i = 0; i < klass->GetIfTableCount(); ++i) { - super_klass.Assign(klass->GetIfTable()->GetInterface(i)); - if (klass->GetClassLoader() != super_klass->GetClassLoader()) { - uint32_t num_methods = super_klass->NumVirtualMethods(); - for (uint32_t j = 0; j < num_methods; ++j) { - ArtMethod* m = klass->GetIfTable()->GetMethodArray(i)->GetElementPtrSize<ArtMethod*>( - j, pointer_size); - ArtMethod* super_m = super_klass->GetVirtualMethod(j, pointer_size); - if (!ResolveTypesOfMethods(self, m) || !ResolveTypesOfMethods(self, super_m)) { - return false; - } + for (int32_t i = 0; i < klass->GetIfTableCount(); ++i) { + super_klass.Assign(klass->GetIfTable()->GetInterface(i)); + if (klass->GetClassLoader() != super_klass->GetClassLoader()) { + uint32_t num_methods = super_klass->NumVirtualMethods(); + for (uint32_t j = 0; j < num_methods; ++j) { + ArtMethod* m = klass->GetIfTable()->GetMethodArray(i)->GetElementPtrSize<ArtMethod*>( + j, pointer_size); + ArtMethod* super_m = super_klass->GetVirtualMethod(j, pointer_size); + if (!ResolveTypesOfMethods(self, m) || !ResolveTypesOfMethods(self, super_m)) { + return false; } } } } - return true; + } + return true; } // Initialize the klass's dependencies recursively before initializing itself. @@ -2548,8 +2553,9 @@ class InitializeClassVisitor : public CompilationVisitor { ObjPtr<mirror::Class> super_class = klass->GetSuperClass(); StackHandleScope<1> hs(self); Handle<mirror::Class> handle_scope_super(hs.NewHandle(super_class)); - if (!NoClinitInDependency(handle_scope_super, self, class_loader)) + if (!NoClinitInDependency(handle_scope_super, self, class_loader)) { return false; + } } uint32_t num_if = klass->NumDirectInterfaces(); @@ -2558,8 +2564,9 @@ class InitializeClassVisitor : public CompilationVisitor { interface = mirror::Class::GetDirectInterface(self, klass.Get(), i); StackHandleScope<1> hs(self); Handle<mirror::Class> handle_interface(hs.NewHandle(interface)); - if (!NoClinitInDependency(handle_interface, self, class_loader)) + if (!NoClinitInDependency(handle_interface, self, class_loader)) { return false; + } } return true; @@ -2681,7 +2688,14 @@ void CompilerDriver::Compile(jobject class_loader, : profile_compilation_info_->DumpInfo(&dex_files)); } - DCHECK(current_dex_to_dex_methods_ == nullptr); + current_dex_to_dex_methods_ = nullptr; + Thread* const self = Thread::Current(); + { + // Clear in case we aren't the first call to Compile. + MutexLock mu(self, dex_to_dex_references_lock_); + dex_to_dex_references_.clear(); + } + for (const DexFile* dex_file : dex_files) { CHECK(dex_file != nullptr); CompileDexFile(class_loader, @@ -2700,7 +2714,7 @@ void CompilerDriver::Compile(jobject class_loader, { // From this point on, we shall not modify dex_to_dex_references_, so // just grab a reference to it that we use without holding the mutex. - MutexLock lock(Thread::Current(), dex_to_dex_references_lock_); + MutexLock lock(self, dex_to_dex_references_lock_); dex_to_dex_references = ArrayRef<DexFileMethodSet>(dex_to_dex_references_); } for (const auto& method_set : dex_to_dex_references) { @@ -2767,13 +2781,7 @@ class CompileClassVisitor : public CompilationVisitor { GetDexToDexCompilationLevel(soa.Self(), *driver, jclass_loader, dex_file, class_def); ClassDataItemIterator it(dex_file, class_data); - // Skip fields - while (it.HasNextStaticField()) { - it.Next(); - } - while (it.HasNextInstanceField()) { - it.Next(); - } + it.SkipAllFields(); bool compilation_enabled = driver->IsClassToCompile( dex_file.StringByTypeIdx(class_def.class_idx_)); @@ -2852,9 +2860,10 @@ void CompilerDriver::AddCompiledMethod(const MethodReference& method_ref, size_t non_relative_linker_patch_count) { DCHECK(GetCompiledMethod(method_ref) == nullptr) << method_ref.dex_file->PrettyMethod(method_ref.dex_method_index); - MethodTable::InsertResult result = compiled_methods_.Insert(method_ref, - /*expected*/ nullptr, - compiled_method); + MethodTable::InsertResult result = compiled_methods_.Insert( + DexFileReference(method_ref.dex_file, method_ref.dex_method_index), + /*expected*/ nullptr, + compiled_method); CHECK(result == MethodTable::kInsertResultSuccess); non_relative_linker_patch_count_.FetchAndAddRelaxed(non_relative_linker_patch_count); DCHECK(GetCompiledMethod(method_ref) != nullptr) @@ -2863,24 +2872,25 @@ void CompilerDriver::AddCompiledMethod(const MethodReference& method_ref, bool CompilerDriver::GetCompiledClass(ClassReference ref, mirror::Class::Status* status) const { DCHECK(status != nullptr); - MutexLock mu(Thread::Current(), compiled_classes_lock_); - ClassStateTable::const_iterator it = compiled_classes_.find(ref); - if (it == compiled_classes_.end()) { + // The table doesn't know if something wasn't inserted. For this case it will return + // kStatusNotReady. To handle this, just assume anything we didn't try to verify is not compiled. + if (!compiled_classes_.Get(DexFileReference(ref.first, ref.second), status) || + *status < mirror::Class::kStatusRetryVerificationAtRuntime) { return false; } - *status = it->second; return true; } void CompilerDriver::RecordClassStatus(ClassReference ref, mirror::Class::Status status) { switch (status) { - case mirror::Class::kStatusNotReady: case mirror::Class::kStatusErrorResolved: case mirror::Class::kStatusErrorUnresolved: + case mirror::Class::kStatusNotReady: + case mirror::Class::kStatusResolved: case mirror::Class::kStatusRetryVerificationAtRuntime: case mirror::Class::kStatusVerified: + case mirror::Class::kStatusSuperclassValidated: case mirror::Class::kStatusInitialized: - case mirror::Class::kStatusResolved: break; // Expected states. default: LOG(FATAL) << "Unexpected class status for class " @@ -2888,20 +2898,35 @@ void CompilerDriver::RecordClassStatus(ClassReference ref, mirror::Class::Status << " of " << status; } - MutexLock mu(Thread::Current(), compiled_classes_lock_); - auto it = compiled_classes_.find(ref); - if (it == compiled_classes_.end()) { - compiled_classes_.Overwrite(ref, status); - } else if (status > it->second) { + ClassStateTable::InsertResult result; + do { + DexFileReference dex_ref(ref.first, ref.second); + mirror::Class::Status existing = mirror::Class::kStatusNotReady; + if (!compiled_classes_.Get(dex_ref, &existing)) { + // Probably a uses library class, bail. + if (kIsDebugBuild) { + // Check to make sure it's not a dex file for an oat file we are compiling since these + // should always succeed. These do not include classes in for used libraries. + for (const DexFile* dex_file : GetDexFilesForOatFile()) { + CHECK_NE(dex_ref.dex_file, dex_file) << dex_ref.dex_file->GetLocation(); + } + } + return; + } + if (existing >= status) { + // Existing status is already better than we expect, break. + break; + } // Update the status if we now have a greater one. This happens with vdex, // which records a class is verified, but does not resolve it. - it->second = status; - } + result = compiled_classes_.Insert(dex_ref, existing, status); + CHECK(result != ClassStateTable::kInsertResultInvalidDexFile); + } while (result != ClassStateTable::kInsertResultSuccess); } CompiledMethod* CompilerDriver::GetCompiledMethod(MethodReference ref) const { CompiledMethod* compiled_method = nullptr; - compiled_methods_.Get(ref, &compiled_method); + compiled_methods_.Get(DexFileReference(ref.dex_file, ref.dex_method_index), &compiled_method); return compiled_method; } @@ -3000,4 +3025,19 @@ void CompilerDriver::FreeThreadPools() { single_thread_pool_.reset(); } +void CompilerDriver::SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files) { + dex_files_for_oat_file_ = dex_files; + for (const DexFile* dex_file : dex_files) { + if (!compiled_classes_.HaveDexFile(dex_file)) { + compiled_classes_.AddDexFile(dex_file, dex_file->NumClassDefs()); + } + } +} + +bool CompilerDriver::CanAssumeVerified(ClassReference ref) const { + mirror::Class::Status existing = mirror::Class::kStatusNotReady; + compiled_classes_.Get(DexFileReference(ref.first, ref.second), &existing); + return existing >= mirror::Class::kStatusVerified; +} + } // namespace art diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h index 38e7d2c686..5771b190f6 100644 --- a/compiler/driver/compiler_driver.h +++ b/compiler/driver/compiler_driver.h @@ -39,7 +39,7 @@ #include "os.h" #include "safe_map.h" #include "thread_pool.h" -#include "utils/atomic_method_ref_map.h" +#include "utils/atomic_dex_ref_map.h" #include "utils/dex_cache_arrays_layout.h" namespace art { @@ -103,32 +103,22 @@ class CompilerDriver { ~CompilerDriver(); // Set dex files that will be stored in the oat file after being compiled. - void SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files) { - dex_files_for_oat_file_ = &dex_files; - } + void SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files); // Get dex file that will be stored in the oat file after being compiled. ArrayRef<const DexFile* const> GetDexFilesForOatFile() const { - return (dex_files_for_oat_file_ != nullptr) - ? ArrayRef<const DexFile* const>(*dex_files_for_oat_file_) - : ArrayRef<const DexFile* const>(); + return ArrayRef<const DexFile* const>(dex_files_for_oat_file_); } void CompileAll(jobject class_loader, const std::vector<const DexFile*>& dex_files, TimingLogger* timings) - REQUIRES(!Locks::mutator_lock_, !compiled_classes_lock_, !dex_to_dex_references_lock_); - - void CompileAll(jobject class_loader, - const std::vector<const DexFile*>& dex_files, - VdexFile* vdex_file, - TimingLogger* timings) - REQUIRES(!Locks::mutator_lock_, !compiled_classes_lock_, !dex_to_dex_references_lock_); + REQUIRES(!Locks::mutator_lock_, !dex_to_dex_references_lock_); // Compile a single Method. void CompileOne(Thread* self, ArtMethod* method, TimingLogger* timings) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!compiled_classes_lock_, !dex_to_dex_references_lock_); + REQUIRES(!dex_to_dex_references_lock_); VerificationResults* GetVerificationResults() const; @@ -159,8 +149,7 @@ class CompilerDriver { std::unique_ptr<const std::vector<uint8_t>> CreateQuickResolutionTrampoline() const; std::unique_ptr<const std::vector<uint8_t>> CreateQuickToInterpreterBridge() const; - bool GetCompiledClass(ClassReference ref, mirror::Class::Status* status) const - REQUIRES(!compiled_classes_lock_); + bool GetCompiledClass(ClassReference ref, mirror::Class::Status* status) const; CompiledMethod* GetCompiledMethod(MethodReference ref) const; size_t GetNonRelativeLinkerPatchCount() const; @@ -260,9 +249,12 @@ class CompilerDriver { // Resolve a method. Returns null on failure, including incompatible class change. ArtMethod* ResolveMethod( - ScopedObjectAccess& soa, Handle<mirror::DexCache> dex_cache, - Handle<mirror::ClassLoader> class_loader, const DexCompilationUnit* mUnit, - uint32_t method_idx, InvokeType invoke_type, bool check_incompatible_class_change = true) + ScopedObjectAccess& soa, + Handle<mirror::DexCache> dex_cache, + Handle<mirror::ClassLoader> class_loader, + const DexCompilationUnit* mUnit, + uint32_t method_idx, + InvokeType invoke_type) REQUIRES_SHARED(Locks::mutator_lock_); void ProcessedInstanceField(bool resolved); @@ -340,8 +332,7 @@ class CompilerDriver { // according to the profile file. bool ShouldVerifyClassBasedOnProfile(const DexFile& dex_file, uint16_t class_idx) const; - void RecordClassStatus(ClassReference ref, mirror::Class::Status status) - REQUIRES(!compiled_classes_lock_); + void RecordClassStatus(ClassReference ref, mirror::Class::Status status); // Checks if the specified method has been verified without failures. Returns // false if the method is not in the verification results (GetVerificationResults). @@ -386,24 +377,13 @@ class CompilerDriver { return profile_compilation_info_; } - private: - // Can `referrer_class` access the resolved `member`? - // Dispatch call to mirror::Class::CanAccessResolvedField or - // mirror::Class::CanAccessResolvedMember depending on the value of - // ArtMember. - template <typename ArtMember> - static bool CanAccessResolvedMember(mirror::Class* referrer_class, - mirror::Class* access_to, - ArtMember* member, - mirror::DexCache* dex_cache, - uint32_t field_idx) - REQUIRES_SHARED(Locks::mutator_lock_); + bool CanAssumeVerified(ClassReference ref) const; private: void PreCompile(jobject class_loader, const std::vector<const DexFile*>& dex_files, TimingLogger* timings) - REQUIRES(!Locks::mutator_lock_, !compiled_classes_lock_); + REQUIRES(!Locks::mutator_lock_); void LoadImageClasses(TimingLogger* timings) REQUIRES(!Locks::mutator_lock_); @@ -424,12 +404,9 @@ class CompilerDriver { // Do fast verification through VerifierDeps if possible. Return whether // verification was successful. - // NO_THREAD_SAFETY_ANALYSIS as the method accesses a guarded value in a - // single-threaded way. bool FastVerify(jobject class_loader, const std::vector<const DexFile*>& dex_files, - TimingLogger* timings) - NO_THREAD_SAFETY_ANALYSIS; + TimingLogger* timings); void Verify(jobject class_loader, const std::vector<const DexFile*>& dex_files, @@ -457,12 +434,12 @@ class CompilerDriver { void InitializeClasses(jobject class_loader, const std::vector<const DexFile*>& dex_files, TimingLogger* timings) - REQUIRES(!Locks::mutator_lock_, !compiled_classes_lock_); + REQUIRES(!Locks::mutator_lock_); void InitializeClasses(jobject class_loader, const DexFile& dex_file, const std::vector<const DexFile*>& dex_files, TimingLogger* timings) - REQUIRES(!Locks::mutator_lock_, !compiled_classes_lock_); + REQUIRES(!Locks::mutator_lock_); void UpdateImageClasses(TimingLogger* timings) REQUIRES(!Locks::mutator_lock_); @@ -500,12 +477,11 @@ class CompilerDriver { std::map<ClassReference, bool> requires_constructor_barrier_ GUARDED_BY(requires_constructor_barrier_lock_); - using ClassStateTable = SafeMap<const ClassReference, mirror::Class::Status>; - // All class references that this compiler has compiled. - mutable Mutex compiled_classes_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; - ClassStateTable compiled_classes_ GUARDED_BY(compiled_classes_lock_); + // All class references that this compiler has compiled. Indexed by class defs. + using ClassStateTable = AtomicDexRefMap<mirror::Class::Status>; + ClassStateTable compiled_classes_; - typedef AtomicMethodRefMap<CompiledMethod*> MethodTable; + typedef AtomicDexRefMap<CompiledMethod*> MethodTable; private: // All method references that this compiler has compiled. @@ -554,7 +530,7 @@ class CompilerDriver { bool support_boot_image_fixup_; // List of dex files that will be stored in the oat file. - const std::vector<const DexFile*>* dex_files_for_oat_file_; + std::vector<const DexFile*> dex_files_for_oat_file_; CompiledMethodStorage compiled_method_storage_; diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc index 4b979d8125..37e17b2a61 100644 --- a/compiler/driver/compiler_driver_test.cc +++ b/compiler/driver/compiler_driver_test.cc @@ -23,6 +23,7 @@ #include "art_method-inl.h" #include "class_linker-inl.h" #include "common_compiler_test.h" +#include "compiler_callbacks.h" #include "dex_file.h" #include "dex_file_types.h" #include "gc/heap.h" @@ -42,10 +43,9 @@ class CompilerDriverTest : public CommonCompilerTest { void CompileAll(jobject class_loader) REQUIRES(!Locks::mutator_lock_) { TimingLogger timings("CompilerDriverTest::CompileAll", false, false); TimingLogger::ScopedTiming t(__FUNCTION__, &timings); - compiler_driver_->CompileAll(class_loader, - GetDexFiles(class_loader), - /* verifier_deps */ nullptr, - &timings); + dex_files_ = GetDexFiles(class_loader); + compiler_driver_->SetDexFilesForOatFile(dex_files_);; + compiler_driver_->CompileAll(class_loader, dex_files_, &timings); t.NewTiming("MakeAllExecutable"); MakeAllExecutable(class_loader); } @@ -98,6 +98,7 @@ class CompilerDriverTest : public CommonCompilerTest { JNIEnv* env_; jclass class_; jmethodID mid_; + std::vector<const DexFile*> dex_files_; }; // Disabled due to 10 second runtime on host @@ -121,10 +122,12 @@ TEST_F(CompilerDriverTest, DISABLED_LARGE_CompileDexLibCore) { EXPECT_TRUE(type != nullptr) << "type_idx=" << i << " " << dex.GetTypeDescriptor(dex.GetTypeId(dex::TypeIndex(i))); } - EXPECT_EQ(dex.NumMethodIds(), dex_cache->NumResolvedMethods()); + EXPECT_TRUE(dex_cache->StaticMethodSize() == dex_cache->NumResolvedMethods() + || dex.NumMethodIds() == dex_cache->NumResolvedMethods()); auto* cl = Runtime::Current()->GetClassLinker(); auto pointer_size = cl->GetImagePointerSize(); for (size_t i = 0; i < dex_cache->NumResolvedMethods(); i++) { + // FIXME: This is outdated for hash-based method array. ArtMethod* method = dex_cache->GetResolvedMethod(i, pointer_size); EXPECT_TRUE(method != nullptr) << "method_idx=" << i << " " << dex.GetMethodDeclaringClassDescriptor(dex.GetMethodId(i)) @@ -136,6 +139,7 @@ TEST_F(CompilerDriverTest, DISABLED_LARGE_CompileDexLibCore) { EXPECT_TRUE(dex_cache->StaticArtFieldSize() == dex_cache->NumResolvedFields() || dex.NumFieldIds() == dex_cache->NumResolvedFields()); for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) { + // FIXME: This is outdated for hash-based field array. ArtField* field = dex_cache->GetResolvedField(i, cl->GetImagePointerSize()); EXPECT_TRUE(field != nullptr) << "field_idx=" << i << " " << dex.GetFieldDeclaringClassDescriptor(dex.GetFieldId(i)) @@ -239,14 +243,10 @@ class CompilerDriverProfileTest : public CompilerDriverTest { ProfileCompilationInfo info; for (const std::unique_ptr<const DexFile>& dex_file : dex_files) { - profile_info_.AddMethodIndex(dex_file->GetLocation(), - dex_file->GetLocationChecksum(), - 1, - dex_file->NumMethodIds()); - profile_info_.AddMethodIndex(dex_file->GetLocation(), - dex_file->GetLocationChecksum(), - 2, - dex_file->NumMethodIds()); + profile_info_.AddMethodIndex(ProfileCompilationInfo::MethodHotness::kFlagHot, + MethodReference(dex_file.get(), 1)); + profile_info_.AddMethodIndex(ProfileCompilationInfo::MethodHotness::kFlagHot, + MethodReference(dex_file.get(), 2)); } return &profile_info_; } @@ -367,6 +367,49 @@ TEST_F(CompilerDriverVerifyTest, VerifyCompilation) { CheckVerifiedClass(class_loader, "LSecond;"); } +// Test that a class of status kStatusRetryVerificationAtRuntime is indeed recorded that way in the +// driver. +// Test that checks that classes can be assumed as verified if unloading mode is enabled and +// the class status is at least verified. +TEST_F(CompilerDriverVerifyTest, RetryVerifcationStatusCheckVerified) { + Thread* const self = Thread::Current(); + jobject class_loader; + std::vector<const DexFile*> dex_files; + const DexFile* dex_file = nullptr; + { + ScopedObjectAccess soa(self); + class_loader = LoadDex("ProfileTestMultiDex"); + ASSERT_NE(class_loader, nullptr); + dex_files = GetDexFiles(class_loader); + ASSERT_GT(dex_files.size(), 0u); + dex_file = dex_files.front(); + } + compiler_driver_->SetDexFilesForOatFile(dex_files); + callbacks_->SetDoesClassUnloading(true, compiler_driver_.get()); + ClassReference ref(dex_file, 0u); + // Test that the status is read from the compiler driver as expected. + for (size_t i = mirror::Class::kStatusRetryVerificationAtRuntime; + i < mirror::Class::kStatusMax; + ++i) { + const mirror::Class::Status expected_status = static_cast<mirror::Class::Status>(i); + // Skip unsupported status that are not supposed to be ever recorded. + if (expected_status == mirror::Class::kStatusVerifyingAtRuntime || + expected_status == mirror::Class::kStatusInitializing) { + continue; + } + compiler_driver_->RecordClassStatus(ref, expected_status); + mirror::Class::Status status = {}; + ASSERT_TRUE(compiler_driver_->GetCompiledClass(ref, &status)); + EXPECT_EQ(status, expected_status); + + // Check that we can assume verified if we are a status that is at least verified. + if (status >= mirror::Class::kStatusVerified) { + // Check that the class can be assumed as verified in the compiler driver. + EXPECT_TRUE(callbacks_->CanAssumeVerified(ref)) << status; + } + } +} + // TODO: need check-cast test (when stub complete & we can throw/catch } // namespace art diff --git a/compiler/driver/compiler_options.cc b/compiler/driver/compiler_options.cc index a4e2083fe4..76f0ae9202 100644 --- a/compiler/driver/compiler_options.cc +++ b/compiler/driver/compiler_options.cc @@ -40,7 +40,7 @@ CompilerOptions::CompilerOptions() implicit_so_checks_(true), implicit_suspend_checks_(false), compile_pic_(false), - verbose_methods_(nullptr), + verbose_methods_(), abort_on_hard_verifier_failure_(false), init_failure_output_(nullptr), dump_cfg_file_name_(""), @@ -55,58 +55,6 @@ CompilerOptions::~CompilerOptions() { // because we don't want to include the PassManagerOptions definition from the header file. } -CompilerOptions::CompilerOptions(CompilerFilter::Filter compiler_filter, - size_t huge_method_threshold, - size_t large_method_threshold, - size_t small_method_threshold, - size_t tiny_method_threshold, - size_t num_dex_methods_threshold, - size_t inline_max_code_units, - const std::vector<const DexFile*>* no_inline_from, - double top_k_profile_threshold, - bool debuggable, - bool generate_debug_info, - bool implicit_null_checks, - bool implicit_so_checks, - bool implicit_suspend_checks, - bool compile_pic, - const std::vector<std::string>* verbose_methods, - std::ostream* init_failure_output, - bool abort_on_hard_verifier_failure, - const std::string& dump_cfg_file_name, - bool dump_cfg_append, - bool force_determinism, - RegisterAllocator::Strategy regalloc_strategy, - const std::vector<std::string>* passes_to_run) - : compiler_filter_(compiler_filter), - huge_method_threshold_(huge_method_threshold), - large_method_threshold_(large_method_threshold), - small_method_threshold_(small_method_threshold), - tiny_method_threshold_(tiny_method_threshold), - num_dex_methods_threshold_(num_dex_methods_threshold), - inline_max_code_units_(inline_max_code_units), - no_inline_from_(no_inline_from), - boot_image_(false), - app_image_(false), - top_k_profile_threshold_(top_k_profile_threshold), - debuggable_(debuggable), - generate_debug_info_(generate_debug_info), - generate_mini_debug_info_(kDefaultGenerateMiniDebugInfo), - generate_build_id_(false), - implicit_null_checks_(implicit_null_checks), - implicit_so_checks_(implicit_so_checks), - implicit_suspend_checks_(implicit_suspend_checks), - compile_pic_(compile_pic), - verbose_methods_(verbose_methods), - abort_on_hard_verifier_failure_(abort_on_hard_verifier_failure), - init_failure_output_(init_failure_output), - dump_cfg_file_name_(dump_cfg_file_name), - dump_cfg_append_(dump_cfg_append), - force_determinism_(force_determinism), - register_allocation_strategy_(regalloc_strategy), - passes_to_run_(passes_to_run) { -} - void CompilerOptions::ParseHugeMethodMax(const StringPiece& option, UsageFn Usage) { ParseUintOption(option, "--huge-method-max", &huge_method_threshold_, Usage); } @@ -204,6 +152,11 @@ bool CompilerOptions::ParseCompilerOption(const StringPiece& option, UsageFn Usa dump_cfg_append_ = true; } else if (option.starts_with("--register-allocation-strategy=")) { ParseRegisterAllocationStrategy(option, Usage); + } else if (option.starts_with("--verbose-methods=")) { + // TODO: rather than switch off compiler logging, make all VLOG(compiler) messages + // conditional on having verbose methods. + gLogVerbosity.compiler = false; + Split(option.substr(strlen("--verbose-methods=")).ToString(), ',', &verbose_methods_); } else { // Option not recognized. return false; diff --git a/compiler/driver/compiler_options.h b/compiler/driver/compiler_options.h index 89c2537476..0a2d54280d 100644 --- a/compiler/driver/compiler_options.h +++ b/compiler/driver/compiler_options.h @@ -52,30 +52,6 @@ class CompilerOptions FINAL { CompilerOptions(); ~CompilerOptions(); - CompilerOptions(CompilerFilter::Filter compiler_filter, - size_t huge_method_threshold, - size_t large_method_threshold, - size_t small_method_threshold, - size_t tiny_method_threshold, - size_t num_dex_methods_threshold, - size_t inline_max_code_units, - const std::vector<const DexFile*>* no_inline_from, - double top_k_profile_threshold, - bool debuggable, - bool generate_debug_info, - bool implicit_null_checks, - bool implicit_so_checks, - bool implicit_suspend_checks, - bool compile_pic, - const std::vector<std::string>* verbose_methods, - std::ostream* init_failure_output, - bool abort_on_hard_verifier_failure, - const std::string& dump_cfg_file_name, - bool dump_cfg_append, - bool force_determinism, - RegisterAllocator::Strategy regalloc_strategy, - const std::vector<std::string>* passes_to_run); - CompilerFilter::Filter GetCompilerFilter() const { return compiler_filter_; } @@ -163,6 +139,10 @@ class CompilerOptions FINAL { return debuggable_; } + void SetDebuggable(bool value) { + debuggable_ = value; + } + bool GetNativeDebuggable() const { return GetDebuggable() && GetGenerateDebugInfo(); } @@ -205,17 +185,21 @@ class CompilerOptions FINAL { return app_image_; } + void DisableAppImage() { + app_image_ = false; + } + // Should the code be compiled as position independent? bool GetCompilePic() const { return compile_pic_; } bool HasVerboseMethods() const { - return verbose_methods_ != nullptr && !verbose_methods_->empty(); + return !verbose_methods_.empty(); } bool IsVerboseMethod(const std::string& pretty_method) const { - for (const std::string& cur_method : *verbose_methods_) { + for (const std::string& cur_method : verbose_methods_) { if (pretty_method.find(cur_method) != std::string::npos) { return true; } @@ -299,7 +283,7 @@ class CompilerOptions FINAL { bool compile_pic_; // Vector of methods to have verbose output enabled for. - const std::vector<std::string>* verbose_methods_; + std::vector<std::string> verbose_methods_; // Abort compilation with an error if we find a class that fails verification with a hard // failure. diff --git a/compiler/elf_builder.h b/compiler/elf_builder.h index 7c02384ff2..2ef9fa1ccb 100644 --- a/compiler/elf_builder.h +++ b/compiler/elf_builder.h @@ -670,6 +670,7 @@ class ElfBuilder FINAL { Elf_Word rodata_size, Elf_Word text_size, Elf_Word bss_size, + Elf_Word bss_methods_offset, Elf_Word bss_roots_offset) { std::string soname(elf_file_path); size_t directory_separator_pos = soname.rfind('/'); @@ -715,9 +716,18 @@ class ElfBuilder FINAL { Elf_Word bss_index = rodata_index + 1u + (text_size != 0 ? 1u : 0u); Elf_Word oatbss = dynstr_.Add("oatbss"); dynsym_.Add(oatbss, bss_index, bss_address, bss_roots_offset, STB_GLOBAL, STT_OBJECT); + DCHECK_LE(bss_methods_offset, bss_roots_offset); + DCHECK_LE(bss_roots_offset, bss_size); + // Add a symbol marking the start of the methods part of the .bss, if not empty. + if (bss_methods_offset != bss_roots_offset) { + Elf_Word bss_methods_address = bss_address + bss_methods_offset; + Elf_Word bss_methods_size = bss_roots_offset - bss_methods_offset; + Elf_Word oatbssroots = dynstr_.Add("oatbssmethods"); + dynsym_.Add( + oatbssroots, bss_index, bss_methods_address, bss_methods_size, STB_GLOBAL, STT_OBJECT); + } // Add a symbol marking the start of the GC roots part of the .bss, if not empty. if (bss_roots_offset != bss_size) { - DCHECK_LT(bss_roots_offset, bss_size); Elf_Word bss_roots_address = bss_address + bss_roots_offset; Elf_Word bss_roots_size = bss_size - bss_roots_offset; Elf_Word oatbssroots = dynstr_.Add("oatbssroots"); diff --git a/compiler/elf_writer.h b/compiler/elf_writer.h index 7baae527ff..a8a5bc32b7 100644 --- a/compiler/elf_writer.h +++ b/compiler/elf_writer.h @@ -55,6 +55,7 @@ class ElfWriter { virtual void PrepareDynamicSection(size_t rodata_size, size_t text_size, size_t bss_size, + size_t bss_methods_offset, size_t bss_roots_offset) = 0; virtual void PrepareDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) = 0; virtual OutputStream* StartRoData() = 0; diff --git a/compiler/elf_writer_quick.cc b/compiler/elf_writer_quick.cc index 738f5a2b29..5d6dd2e1d7 100644 --- a/compiler/elf_writer_quick.cc +++ b/compiler/elf_writer_quick.cc @@ -22,7 +22,6 @@ #include "base/casts.h" #include "base/logging.h" -#include "base/stl_util.h" #include "compiled_method.h" #include "debug/elf_debug_writer.h" #include "debug/method_debug_info.h" @@ -80,7 +79,7 @@ class DebugInfoTask : public Task { const InstructionSetFeatures* instruction_set_features_; size_t rodata_section_size_; size_t text_section_size_; - const ArrayRef<const debug::MethodDebugInfo>& method_infos_; + const ArrayRef<const debug::MethodDebugInfo> method_infos_; std::vector<uint8_t> result_; }; @@ -97,6 +96,7 @@ class ElfWriterQuick FINAL : public ElfWriter { void PrepareDynamicSection(size_t rodata_size, size_t text_size, size_t bss_size, + size_t bss_methods_offset, size_t bss_roots_offset) OVERRIDE; void PrepareDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) OVERRIDE; OutputStream* StartRoData() OVERRIDE; @@ -136,15 +136,15 @@ std::unique_ptr<ElfWriter> CreateElfWriterQuick(InstructionSet instruction_set, const CompilerOptions* compiler_options, File* elf_file) { if (Is64BitInstructionSet(instruction_set)) { - return MakeUnique<ElfWriterQuick<ElfTypes64>>(instruction_set, - features, - compiler_options, - elf_file); + return std::make_unique<ElfWriterQuick<ElfTypes64>>(instruction_set, + features, + compiler_options, + elf_file); } else { - return MakeUnique<ElfWriterQuick<ElfTypes32>>(instruction_set, - features, - compiler_options, - elf_file); + return std::make_unique<ElfWriterQuick<ElfTypes32>>(instruction_set, + features, + compiler_options, + elf_file); } } @@ -160,7 +160,8 @@ ElfWriterQuick<ElfTypes>::ElfWriterQuick(InstructionSet instruction_set, rodata_size_(0u), text_size_(0u), bss_size_(0u), - output_stream_(MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(elf_file))), + output_stream_( + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(elf_file))), builder_(new ElfBuilder<ElfTypes>(instruction_set, features, output_stream_.get())) {} template <typename ElfTypes> @@ -178,6 +179,7 @@ template <typename ElfTypes> void ElfWriterQuick<ElfTypes>::PrepareDynamicSection(size_t rodata_size, size_t text_size, size_t bss_size, + size_t bss_methods_offset, size_t bss_roots_offset) { DCHECK_EQ(rodata_size_, 0u); rodata_size_ = rodata_size; @@ -189,6 +191,7 @@ void ElfWriterQuick<ElfTypes>::PrepareDynamicSection(size_t rodata_size, rodata_size_, text_size_, bss_size_, + bss_methods_offset, bss_roots_offset); } diff --git a/compiler/exception_test.cc b/compiler/exception_test.cc index dc880b089e..0b3ca69846 100644 --- a/compiler/exception_test.cc +++ b/compiler/exception_test.cc @@ -17,6 +17,7 @@ #include <memory> #include "base/arena_allocator.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "class_linker.h" #include "common_runtime_test.h" @@ -101,12 +102,14 @@ class ExceptionTest : public CommonRuntimeTest { CHECK_ALIGNED(stack_maps_offset, 2); } - method_f_ = my_klass_->FindVirtualMethod("f", "()I", kRuntimePointerSize); + method_f_ = my_klass_->FindClassMethod("f", "()I", kRuntimePointerSize); ASSERT_TRUE(method_f_ != nullptr); + ASSERT_FALSE(method_f_->IsDirect()); method_f_->SetEntryPointFromQuickCompiledCode(code_ptr); - method_g_ = my_klass_->FindVirtualMethod("g", "(I)V", kRuntimePointerSize); + method_g_ = my_klass_->FindClassMethod("g", "(I)V", kRuntimePointerSize); ASSERT_TRUE(method_g_ != nullptr); + ASSERT_FALSE(method_g_->IsDirect()); method_g_->SetEntryPointFromQuickCompiledCode(code_ptr); } @@ -170,7 +173,7 @@ TEST_F(ExceptionTest, StackTraceElement) { Runtime* r = Runtime::Current(); r->SetInstructionSet(kRuntimeISA); ArtMethod* save_method = r->CreateCalleeSaveMethod(); - r->SetCalleeSaveMethod(save_method, Runtime::kSaveAllCalleeSaves); + r->SetCalleeSaveMethod(save_method, CalleeSaveType::kSaveAllCalleeSaves); QuickMethodFrameInfo frame_info = r->GetRuntimeMethodFrameInfo(save_method); ASSERT_EQ(kStackAlignment, 16U); diff --git a/compiler/image_test.cc b/compiler/image_test.cc index 9d7aff769b..252fdd67e1 100644 --- a/compiler/image_test.cc +++ b/compiler/image_test.cc @@ -113,9 +113,9 @@ TEST_F(ImageTest, TestDefaultMethods) { mirror::Class* iface_klass = class_linker_->LookupClass( self, "LIface;", ObjPtr<mirror::ClassLoader>()); ASSERT_NE(nullptr, iface_klass); - ArtMethod* origin = iface_klass->FindDeclaredVirtualMethod( - "defaultMethod", "()V", pointer_size); + ArtMethod* origin = iface_klass->FindInterfaceMethod("defaultMethod", "()V", pointer_size); ASSERT_NE(nullptr, origin); + ASSERT_TRUE(origin->GetDeclaringClass() == iface_klass); const void* code = origin->GetEntryPointFromQuickCompiledCodePtrSize(pointer_size); // The origin method should have a pointer to quick code ASSERT_NE(nullptr, code); @@ -134,9 +134,11 @@ TEST_F(ImageTest, TestDefaultMethods) { mirror::Class* iterable_klass = class_linker_->LookupClass( self, "Ljava/lang/Iterable;", ObjPtr<mirror::ClassLoader>()); ASSERT_NE(nullptr, iterable_klass); - origin = iterable_klass->FindDeclaredVirtualMethod( + origin = iterable_klass->FindClassMethod( "forEach", "(Ljava/util/function/Consumer;)V", pointer_size); ASSERT_NE(nullptr, origin); + ASSERT_FALSE(origin->IsDirect()); + ASSERT_TRUE(origin->GetDeclaringClass() == iterable_klass); code = origin->GetEntryPointFromQuickCompiledCodePtrSize(pointer_size); // the origin method should have a pointer to quick code ASSERT_NE(nullptr, code); diff --git a/compiler/image_test.h b/compiler/image_test.h index 2f15ff4815..a2d5a05604 100644 --- a/compiler/image_test.h +++ b/compiler/image_test.h @@ -84,9 +84,10 @@ class ImageTest : public CommonCompilerTest { void SetUpRuntimeOptions(RuntimeOptions* options) OVERRIDE { CommonCompilerTest::SetUpRuntimeOptions(options); - callbacks_.reset(new QuickCompilerCallbacks( - verification_results_.get(), - CompilerCallbacks::CallbackMode::kCompileBootImage)); + QuickCompilerCallbacks* new_callbacks = + new QuickCompilerCallbacks(CompilerCallbacks::CallbackMode::kCompileBootImage); + new_callbacks->SetVerificationResults(verification_results_.get()); + callbacks_.reset(new_callbacks); options->push_back(std::make_pair("compilercallbacks", callbacks_.get())); } @@ -213,14 +214,15 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, /*compile_app_image*/false, storage_mode, oat_filename_vector, - dex_file_to_oat_index_map)); + dex_file_to_oat_index_map, + /*dirty_image_objects*/nullptr)); { { jobject class_loader = nullptr; TimingLogger timings("ImageTest::WriteRead", false, false); TimingLogger::ScopedTiming t("CompileAll", &timings); driver->SetDexFilesForOatFile(class_path); - driver->CompileAll(class_loader, class_path, /* verifier_deps */ nullptr, &timings); + driver->CompileAll(class_loader, class_path, &timings); t.NewTiming("WriteElf"); SafeMap<std::string, std::string> key_value_store; @@ -290,9 +292,9 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, if (kIsVdexEnabled) { for (size_t i = 0, size = vdex_files.size(); i != size; ++i) { - std::unique_ptr<BufferedOutputStream> vdex_out( - MakeUnique<BufferedOutputStream>( - MakeUnique<FileOutputStream>(vdex_files[i].GetFile()))); + std::unique_ptr<BufferedOutputStream> vdex_out = + std::make_unique<BufferedOutputStream>( + std::make_unique<FileOutputStream>(vdex_files[i].GetFile())); oat_writers[i]->WriteVerifierDeps(vdex_out.get(), nullptr); oat_writers[i]->WriteChecksumsAndVdexHeader(vdex_out.get()); } @@ -311,6 +313,7 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, elf_writer->PrepareDynamicSection(rodata_size, text_size, oat_writer->GetBssSize(), + oat_writer->GetBssMethodsOffset(), oat_writer->GetBssRootsOffset()); writer->UpdateOatFileLayout(i, diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 4d6db4745f..3eda6e1bb6 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -27,6 +27,8 @@ #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" +#include "base/enums.h" #include "base/logging.h" #include "base/unix_file/fd_file.h" #include "class_linker-inl.h" @@ -42,12 +44,14 @@ #include "gc/accounting/space_bitmap-inl.h" #include "gc/collector/concurrent_copying.h" #include "gc/heap.h" +#include "gc/heap-visit-objects-inl.h" #include "gc/space/large_object_space.h" #include "gc/space/space-inl.h" +#include "gc/verification.h" #include "globals.h" +#include "handle_scope-inl.h" #include "image.h" #include "imt_conflict_table.h" -#include "intern_table.h" #include "jni_internal.h" #include "linear_alloc.h" #include "lock_word.h" @@ -68,7 +72,6 @@ #include "oat_file_manager.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "handle_scope-inl.h" #include "utils/dex_cache_arrays_layout-inl.h" using ::art::mirror::Class; @@ -115,19 +118,17 @@ bool ImageWriter::IsInBootOatFile(const void* ptr) const { return false; } -static void ClearDexFileCookieCallback(Object* obj, void* arg ATTRIBUTE_UNUSED) - REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK(obj != nullptr); - Class* klass = obj->GetClass(); - if (klass == WellKnownClasses::ToClass(WellKnownClasses::dalvik_system_DexFile)) { - ArtField* field = jni::DecodeArtField(WellKnownClasses::dalvik_system_DexFile_cookie); - // Null out the cookie to enable determinism. b/34090128 - field->SetObject</*kTransactionActive*/false>(obj, nullptr); - } -} - static void ClearDexFileCookies() REQUIRES_SHARED(Locks::mutator_lock_) { - Runtime::Current()->GetHeap()->VisitObjects(ClearDexFileCookieCallback, nullptr); + auto visitor = [](Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(obj != nullptr); + Class* klass = obj->GetClass(); + if (klass == WellKnownClasses::ToClass(WellKnownClasses::dalvik_system_DexFile)) { + ArtField* field = jni::DecodeArtField(WellKnownClasses::dalvik_system_DexFile_cookie); + // Null out the cookie to enable determinism. b/34090128 + field->SetObject</*kTransactionActive*/false>(obj, nullptr); + } + }; + Runtime::Current()->GetHeap()->VisitObjects(visitor); } bool ImageWriter::PrepareImageAddressSpace() { @@ -396,12 +397,18 @@ void ImageWriter::SetImageBinSlot(mirror::Object* object, BinSlot bin_slot) { // Before we stomp over the lock word, save the hash code for later. LockWord lw(object->GetLockWord(false)); switch (lw.GetState()) { - case LockWord::kFatLocked: { - LOG(FATAL) << "Fat locked object " << object << " found during object copy"; - break; - } + case LockWord::kFatLocked: + FALLTHROUGH_INTENDED; case LockWord::kThinLocked: { - LOG(FATAL) << "Thin locked object " << object << " found during object copy"; + std::ostringstream oss; + bool thin = (lw.GetState() == LockWord::kThinLocked); + oss << (thin ? "Thin" : "Fat") + << " locked object " << object << "(" << object->PrettyTypeOf() + << ") found during object copy"; + if (thin) { + oss << ". Lock owner:" << lw.ThinLockOwner(); + } + LOG(FATAL) << oss.str(); break; } case LockWord::kUnlocked: @@ -471,6 +478,11 @@ void ImageWriter::PrepareDexCacheArraySlots() { start + layout.MethodTypesOffset(), dex_cache); } + if (dex_cache->GetResolvedCallSites() != nullptr) { + AddDexCacheArrayRelocation(dex_cache->GetResolvedCallSites(), + start + layout.CallSitesOffset(), + dex_cache); + } } } @@ -567,7 +579,12 @@ void ImageWriter::AssignImageBinSlot(mirror::Object* object, size_t oat_index) { } } - if (klass->GetStatus() == Class::kStatusInitialized) { + // Move known dirty objects into their own sections. This includes: + // - classes with dirty static fields. + if (dirty_image_objects_ != nullptr && + dirty_image_objects_->find(klass->PrettyDescriptor()) != dirty_image_objects_->end()) { + bin = kBinKnownDirty; + } else if (klass->GetStatus() == Class::kStatusInitialized) { bin = kBinClassInitialized; // If the class's static fields are all final, put it into a separate bin @@ -724,16 +741,82 @@ bool ImageWriter::IsBootClassLoaderNonImageClass(mirror::Class* klass) { return IsBootClassLoaderClass(klass) && !IsInBootImage(klass); } +// This visitor follows the references of an instance, recursively then prune this class +// if a type of any field is pruned. +class ImageWriter::PruneObjectReferenceVisitor { + public: + PruneObjectReferenceVisitor(ImageWriter* image_writer, + bool* early_exit, + std::unordered_set<mirror::Object*>* visited, + bool* result) + : image_writer_(image_writer), early_exit_(early_exit), visited_(visited), result_(result) {} + + ALWAYS_INLINE void VisitRootIfNonNull( + mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const + REQUIRES_SHARED(Locks::mutator_lock_) { } + + ALWAYS_INLINE void VisitRoot( + mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const + REQUIRES_SHARED(Locks::mutator_lock_) { } + + ALWAYS_INLINE void operator() (ObjPtr<mirror::Object> obj, + MemberOffset offset, + bool is_static ATTRIBUTE_UNUSED) const + REQUIRES_SHARED(Locks::mutator_lock_) { + mirror::Object* ref = + obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(offset); + if (ref == nullptr || visited_->find(ref) != visited_->end()) { + return; + } + + ObjPtr<mirror::Class> klass = ref->IsClass() ? ref->AsClass() : ref->GetClass(); + if (klass == mirror::Method::StaticClass() || klass == mirror::Constructor::StaticClass()) { + // Prune all classes using reflection because the content they held will not be fixup. + *result_ = true; + } + + // Record the object visited in case of circular reference. + visited_->emplace(ref); + if (ref->IsClass()) { + *result_ = *result_ || + image_writer_->PruneAppImageClassInternal(ref->AsClass(), early_exit_, visited_); + } else { + *result_ = *result_ || + image_writer_->PruneAppImageClassInternal(klass, early_exit_, visited_); + ref->VisitReferences(*this, *this); + } + // Clean up before exit for next call of this function. + visited_->erase(ref); + } + + ALWAYS_INLINE void operator() (ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED, + ObjPtr<mirror::Reference> ref) const + REQUIRES_SHARED(Locks::mutator_lock_) { + operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false); + } + + ALWAYS_INLINE bool GetResult() const { + return result_; + } + + private: + ImageWriter* image_writer_; + bool* early_exit_; + std::unordered_set<mirror::Object*>* visited_; + bool* const result_; +}; + + bool ImageWriter::PruneAppImageClass(ObjPtr<mirror::Class> klass) { bool early_exit = false; - std::unordered_set<mirror::Class*> visited; + std::unordered_set<mirror::Object*> visited; return PruneAppImageClassInternal(klass, &early_exit, &visited); } bool ImageWriter::PruneAppImageClassInternal( ObjPtr<mirror::Class> klass, bool* early_exit, - std::unordered_set<mirror::Class*>* visited) { + std::unordered_set<mirror::Object*>* visited) { DCHECK(early_exit != nullptr); DCHECK(visited != nullptr); DCHECK(compile_app_image_); @@ -794,9 +877,18 @@ bool ImageWriter::PruneAppImageClassInternal( &my_early_exit, visited); } else { - result = result || PruneAppImageClassInternal(ref->GetClass(), + mirror::Class* type = ref->GetClass(); + result = result || PruneAppImageClassInternal(type, &my_early_exit, visited); + if (!result) { + // For non-class case, also go through all the types mentioned by it's fields' + // references recursively to decide whether to keep this class. + bool tmp = false; + PruneObjectReferenceVisitor visitor(this, &my_early_exit, visited, &tmp); + ref->VisitReferences(visitor, visitor); + result = result || tmp; + } } } field_offset = MemberOffset(field_offset.Uint32Value() + @@ -807,7 +899,7 @@ bool ImageWriter::PruneAppImageClassInternal( &my_early_exit, visited); // Remove the class if the dex file is not in the set of dex files. This happens for classes that - // are from uses library if there is no profile. b/30688277 + // are from uses-library if there is no profile. b/30688277 mirror::DexCache* dex_cache = klass->GetDexCache(); if (dex_cache != nullptr) { result = result || @@ -936,34 +1028,58 @@ void ImageWriter::PruneAndPreloadDexCache(ObjPtr<mirror::DexCache> dex_cache, Runtime* runtime = Runtime::Current(); ClassLinker* class_linker = runtime->GetClassLinker(); - ArtMethod* resolution_method = runtime->GetResolutionMethod(); const DexFile& dex_file = *dex_cache->GetDexFile(); // Prune methods. - ArtMethod** resolved_methods = dex_cache->GetResolvedMethods(); - for (size_t i = 0, num = dex_cache->NumResolvedMethods(); i != num; ++i) { - ArtMethod* method = - mirror::DexCache::GetElementPtrSize(resolved_methods, i, target_ptr_size_); - DCHECK(method != nullptr) << "Expected resolution method instead of null method"; - mirror::Class* declaring_class = method->GetDeclaringClass(); - // Copied methods may be held live by a class which was not an image class but have a - // declaring class which is an image class. Set it to the resolution method to be safe and - // prevent dangling pointers. - if (method->IsCopied() || !KeepClass(declaring_class)) { - mirror::DexCache::SetElementPtrSize(resolved_methods, - i, - resolution_method, - target_ptr_size_); - } else if (kIsDebugBuild) { - // Check that the class is still in the classes table. - ReaderMutexLock mu(Thread::Current(), *Locks::classlinker_classes_lock_); - CHECK(class_linker->ClassInClassTable(declaring_class)) << "Class " - << Class::PrettyClass(declaring_class) << " not in class linker table"; + mirror::MethodDexCacheType* resolved_methods = dex_cache->GetResolvedMethods(); + dex::TypeIndex last_class_idx; // Initialized to invalid index. + ObjPtr<mirror::Class> last_class = nullptr; + for (size_t i = 0, num = dex_cache->GetDexFile()->NumMethodIds(); i != num; ++i) { + uint32_t slot_idx = dex_cache->MethodSlotIndex(i); + auto pair = + mirror::DexCache::GetNativePairPtrSize(resolved_methods, slot_idx, target_ptr_size_); + uint32_t stored_index = pair.index; + ArtMethod* method = pair.object; + if (method != nullptr && i > stored_index) { + continue; // Already checked. + } + // Check if the referenced class is in the image. Note that we want to check the referenced + // class rather than the declaring class to preserve the semantics, i.e. using a MethodId + // results in resolving the referenced class and that can for example throw OOME. + const DexFile::MethodId& method_id = dex_file.GetMethodId(i); + if (method_id.class_idx_ != last_class_idx) { + last_class_idx = method_id.class_idx_; + last_class = class_linker->LookupResolvedType( + dex_file, last_class_idx, dex_cache, class_loader); + if (last_class != nullptr && !KeepClass(last_class)) { + last_class = nullptr; + } + } + if (method == nullptr || i < stored_index) { + if (last_class != nullptr) { + const char* name = dex_file.StringDataByIdx(method_id.name_idx_); + Signature signature = dex_file.GetMethodSignature(method_id); + if (last_class->IsInterface()) { + method = last_class->FindInterfaceMethod(name, signature, target_ptr_size_); + } else { + method = last_class->FindClassMethod(name, signature, target_ptr_size_); + } + if (method != nullptr) { + // If the referenced class is in the image, the defining class must also be there. + DCHECK(KeepClass(method->GetDeclaringClass())); + dex_cache->SetResolvedMethod(i, method, target_ptr_size_); + } + } + } else { + DCHECK_EQ(i, stored_index); + if (last_class == nullptr) { + dex_cache->ClearResolvedMethod(stored_index, target_ptr_size_); + } } } // Prune fields and make the contents of the field array deterministic. mirror::FieldDexCacheType* resolved_fields = dex_cache->GetResolvedFields(); - dex::TypeIndex last_class_idx; // Initialized to invalid index. - ObjPtr<mirror::Class> last_class = nullptr; + last_class_idx = dex::TypeIndex(); // Initialized to invalid index. + last_class = nullptr; for (size_t i = 0, end = dex_file.NumFieldIds(); i < end; ++i) { uint32_t slot_idx = dex_cache->FieldSlotIndex(i); auto pair = mirror::DexCache::GetNativePairPtrSize(resolved_fields, slot_idx, target_ptr_size_); @@ -1042,9 +1158,22 @@ void ImageWriter::PruneNonImageClasses() { Thread* self = Thread::Current(); ScopedAssertNoThreadSuspension sa(__FUNCTION__); - // Clear class table strong roots so that dex caches can get pruned. We require pruning the class - // path dex caches. - class_linker->ClearClassTableStrongRoots(); + // Prune uses-library dex caches. Only prune the uses-library dex caches since we want to make + // sure the other ones don't get unloaded before the OatWriter runs. + class_linker->VisitClassTables( + [&](ClassTable* table) REQUIRES_SHARED(Locks::mutator_lock_) { + table->RemoveStrongRoots( + [&](GcRoot<mirror::Object> root) REQUIRES_SHARED(Locks::mutator_lock_) { + ObjPtr<mirror::Object> obj = root.Read(); + if (obj->IsDexCache()) { + // Return true if the dex file is not one of the ones in the map. + return dex_file_oat_index_map_.find(obj->AsDexCache()->GetDexFile()) == + dex_file_oat_index_map_.end(); + } + // Return false to avoid removing. + return false; + }); + }); // Remove the undesired classes from the class roots. ObjPtr<mirror::ClassLoader> class_loader; @@ -1081,21 +1210,19 @@ void ImageWriter::PruneNonImageClasses() { void ImageWriter::CheckNonImageClassesRemoved() { if (compiler_driver_.GetImageClasses() != nullptr) { + auto visitor = [&](Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) { + if (obj->IsClass() && !IsInBootImage(obj)) { + Class* klass = obj->AsClass(); + if (!KeepClass(klass)) { + DumpImageClasses(); + std::string temp; + CHECK(KeepClass(klass)) + << Runtime::Current()->GetHeap()->GetVerification()->FirstPathFromRootSet(klass); + } + } + }; gc::Heap* heap = Runtime::Current()->GetHeap(); - heap->VisitObjects(CheckNonImageClassesRemovedCallback, this); - } -} - -void ImageWriter::CheckNonImageClassesRemovedCallback(Object* obj, void* arg) { - ImageWriter* image_writer = reinterpret_cast<ImageWriter*>(arg); - if (obj->IsClass() && !image_writer->IsInBootImage(obj)) { - Class* klass = obj->AsClass(); - if (!image_writer->KeepClass(klass)) { - image_writer->DumpImageClasses(); - std::string temp; - CHECK(image_writer->KeepClass(klass)) << klass->GetDescriptor(&temp) - << " " << klass->PrettyDescriptor(); - } + heap->VisitObjects(visitor); } } @@ -1437,26 +1564,6 @@ void ImageWriter::AssignMethodOffset(ArtMethod* method, offset += ArtMethod::Size(target_ptr_size_); } -void ImageWriter::EnsureBinSlotAssignedCallback(mirror::Object* obj, void* arg) { - ImageWriter* writer = reinterpret_cast<ImageWriter*>(arg); - DCHECK(writer != nullptr); - if (!Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(obj)) { - CHECK(writer->IsImageBinSlotAssigned(obj)) << mirror::Object::PrettyTypeOf(obj) << " " << obj; - } -} - -void ImageWriter::DeflateMonitorCallback(mirror::Object* obj, void* arg ATTRIBUTE_UNUSED) { - Monitor::Deflate(Thread::Current(), obj); -} - -void ImageWriter::UnbinObjectsIntoOffsetCallback(mirror::Object* obj, void* arg) { - ImageWriter* writer = reinterpret_cast<ImageWriter*>(arg); - DCHECK(writer != nullptr); - if (!writer->IsInBootImage(obj)) { - writer->UnbinObjectsIntoOffset(obj); - } -} - void ImageWriter::UnbinObjectsIntoOffset(mirror::Object* obj) { DCHECK(!IsInBootImage(obj)); CHECK(obj != nullptr); @@ -1572,13 +1679,13 @@ void ImageWriter::CalculateNewObjectOffsets() { image_methods_[ImageHeader::kImtConflictMethod] = runtime->GetImtConflictMethod(); image_methods_[ImageHeader::kImtUnimplementedMethod] = runtime->GetImtUnimplementedMethod(); image_methods_[ImageHeader::kSaveAllCalleeSavesMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveAllCalleeSaves); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveAllCalleeSaves); image_methods_[ImageHeader::kSaveRefsOnlyMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveRefsOnly); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsOnly); image_methods_[ImageHeader::kSaveRefsAndArgsMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs); image_methods_[ImageHeader::kSaveEverythingMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveEverything); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything); // Visit image methods first to have the main runtime methods in the first image. for (auto* m : image_methods_) { CHECK(m != nullptr); @@ -1591,7 +1698,12 @@ void ImageWriter::CalculateNewObjectOffsets() { // Deflate monitors before we visit roots since deflating acquires the monitor lock. Acquiring // this lock while holding other locks may cause lock order violations. - heap->VisitObjects(DeflateMonitorCallback, this); + { + auto deflate_monitor = [](mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) { + Monitor::Deflate(Thread::Current(), obj); + }; + heap->VisitObjects(deflate_monitor); + } // Work list of <object, oat_index> for objects. Everything on the stack must already be // assigned a bin slot. @@ -1653,7 +1765,15 @@ void ImageWriter::CalculateNewObjectOffsets() { } // Verify that all objects have assigned image bin slots. - heap->VisitObjects(EnsureBinSlotAssignedCallback, this); + { + auto ensure_bin_slots_assigned = [&](mirror::Object* obj) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (!Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(obj)) { + CHECK(IsImageBinSlotAssigned(obj)) << mirror::Object::PrettyTypeOf(obj) << " " << obj; + } + }; + heap->VisitObjects(ensure_bin_slots_assigned); + } // Calculate size of the dex cache arrays slot and prepare offsets. PrepareDexCacheArraySlots(); @@ -1717,7 +1837,15 @@ void ImageWriter::CalculateNewObjectOffsets() { } // Transform each object's bin slot into an offset which will be used to do the final copy. - heap->VisitObjects(UnbinObjectsIntoOffsetCallback, this); + { + auto unbin_objects_into_offset = [&](mirror::Object* obj) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (!IsInBootImage(obj)) { + UnbinObjectsIntoOffset(obj); + } + }; + heap->VisitObjects(unbin_objects_into_offset); + } size_t i = 0; for (ImageInfo& image_info : image_infos_) { @@ -2024,8 +2152,11 @@ void ImageWriter::CopyAndFixupNativeData(size_t oat_index) { } void ImageWriter::CopyAndFixupObjects() { - gc::Heap* heap = Runtime::Current()->GetHeap(); - heap->VisitObjects(CopyAndFixupObjectsCallback, this); + auto visitor = [&](Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(obj != nullptr); + CopyAndFixupObject(obj); + }; + Runtime::Current()->GetHeap()->VisitObjects(visitor); // Fix up the object previously had hash codes. for (const auto& hash_pair : saved_hashcode_map_) { Object* obj = hash_pair.first; @@ -2035,12 +2166,6 @@ void ImageWriter::CopyAndFixupObjects() { saved_hashcode_map_.clear(); } -void ImageWriter::CopyAndFixupObjectsCallback(Object* obj, void* arg) { - DCHECK(obj != nullptr); - DCHECK(arg != nullptr); - reinterpret_cast<ImageWriter*>(arg)->CopyAndFixupObject(obj); -} - void ImageWriter::FixupPointerArray(mirror::Object* dst, mirror::PointerArray* arr, mirror::Class* klass, @@ -2311,17 +2436,19 @@ void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, orig_dex_cache->FixupResolvedTypes(NativeCopyLocation(orig_types, orig_dex_cache), fixup_visitor); } - ArtMethod** orig_methods = orig_dex_cache->GetResolvedMethods(); + mirror::MethodDexCacheType* orig_methods = orig_dex_cache->GetResolvedMethods(); if (orig_methods != nullptr) { copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::ResolvedMethodsOffset(), NativeLocationInImage(orig_methods), PointerSize::k64); - ArtMethod** copy_methods = NativeCopyLocation(orig_methods, orig_dex_cache); + mirror::MethodDexCacheType* copy_methods = NativeCopyLocation(orig_methods, orig_dex_cache); for (size_t i = 0, num = orig_dex_cache->NumResolvedMethods(); i != num; ++i) { - ArtMethod* orig = mirror::DexCache::GetElementPtrSize(orig_methods, i, target_ptr_size_); + mirror::MethodDexCachePair orig_pair = + mirror::DexCache::GetNativePairPtrSize(orig_methods, i, target_ptr_size_); // NativeLocationInImage also handles runtime methods since these have relocation info. - ArtMethod* copy = NativeLocationInImage(orig); - mirror::DexCache::SetElementPtrSize(copy_methods, i, copy, target_ptr_size_); + mirror::MethodDexCachePair copy_pair(NativeLocationInImage(orig_pair.object), + orig_pair.index); + mirror::DexCache::SetNativePairPtrSize(copy_methods, i, copy_pair, target_ptr_size_); } } mirror::FieldDexCacheType* orig_fields = orig_dex_cache->GetResolvedFields(); @@ -2462,7 +2589,8 @@ void ImageWriter::CopyAndFixupMethod(ArtMethod* orig, CopyReference(copy->GetDeclaringClassAddressWithoutBarrier(), orig->GetDeclaringClassUnchecked()); - ArtMethod** orig_resolved_methods = orig->GetDexCacheResolvedMethods(target_ptr_size_); + mirror::MethodDexCacheType* orig_resolved_methods = + orig->GetDexCacheResolvedMethods(target_ptr_size_); copy->SetDexCacheResolvedMethods(NativeLocationInImage(orig_resolved_methods), target_ptr_size_); // OatWriter replaces the code_ with an offset value. Here we re-adjust to a pointer relative to @@ -2482,8 +2610,8 @@ void ImageWriter::CopyAndFixupMethod(ArtMethod* orig, GetOatAddress(kOatAddressQuickResolutionTrampoline), target_ptr_size_); } else { bool found_one = false; - for (size_t i = 0; i < static_cast<size_t>(Runtime::kLastCalleeSaveType); ++i) { - auto idx = static_cast<Runtime::CalleeSaveType>(i); + for (size_t i = 0; i < static_cast<size_t>(CalleeSaveType::kLastCalleeSaveType); ++i) { + auto idx = static_cast<CalleeSaveType>(i); if (runtime->HasCalleeSaveMethod(idx) && runtime->GetCalleeSaveMethod(idx) == orig) { found_one = true; break; @@ -2651,7 +2779,8 @@ ImageWriter::ImageWriter( bool compile_app_image, ImageHeader::StorageMode image_storage_mode, const std::vector<const char*>& oat_filenames, - const std::unordered_map<const DexFile*, size_t>& dex_file_oat_index_map) + const std::unordered_map<const DexFile*, size_t>& dex_file_oat_index_map, + const std::unordered_set<std::string>* dirty_image_objects) : compiler_driver_(compiler_driver), global_image_begin_(reinterpret_cast<uint8_t*>(image_begin)), image_objects_offset_begin_(0), @@ -2663,7 +2792,8 @@ ImageWriter::ImageWriter( clean_methods_(0u), image_storage_mode_(image_storage_mode), oat_filenames_(oat_filenames), - dex_file_oat_index_map_(dex_file_oat_index_map) { + dex_file_oat_index_map_(dex_file_oat_index_map), + dirty_image_objects_(dirty_image_objects) { CHECK_NE(image_begin, 0U); std::fill_n(image_methods_, arraysize(image_methods_), nullptr); CHECK_EQ(compile_app_image, !Runtime::Current()->GetHeap()->GetBootImageSpaces().empty()) diff --git a/compiler/image_writer.h b/compiler/image_writer.h index 2283b39773..ee8a45d380 100644 --- a/compiler/image_writer.h +++ b/compiler/image_writer.h @@ -36,6 +36,7 @@ #include "class_table.h" #include "driver/compiler_driver.h" #include "image.h" +#include "intern_table.h" #include "lock_word.h" #include "mem_map.h" #include "mirror/dex_cache.h" @@ -74,7 +75,8 @@ class ImageWriter FINAL { bool compile_app_image, ImageHeader::StorageMode image_storage_mode, const std::vector<const char*>& oat_filenames, - const std::unordered_map<const DexFile*, size_t>& dex_file_oat_index_map); + const std::unordered_map<const DexFile*, size_t>& dex_file_oat_index_map, + const std::unordered_set<std::string>* dirty_image_objects); bool PrepareImageAddressSpace(); @@ -106,19 +108,6 @@ class ImageWriter FINAL { ArtMethod* GetImageMethodAddress(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_); - template <typename PtrType> - PtrType GetDexCacheArrayElementImageAddress(const DexFile* dex_file, uint32_t offset) - const REQUIRES_SHARED(Locks::mutator_lock_) { - auto oat_it = dex_file_oat_index_map_.find(dex_file); - DCHECK(oat_it != dex_file_oat_index_map_.end()); - const ImageInfo& image_info = GetImageInfo(oat_it->second); - auto it = image_info.dex_cache_array_starts_.find(dex_file); - DCHECK(it != image_info.dex_cache_array_starts_.end()); - return reinterpret_cast<PtrType>( - image_info.image_begin_ + image_info.bin_slot_offsets_[kBinDexCacheArray] + - it->second + offset); - } - size_t GetOatFileOffset(size_t oat_index) const { return GetImageInfo(oat_index).oat_offset_; } @@ -171,6 +160,7 @@ class ImageWriter FINAL { // Classify different kinds of bins that objects end up getting packed into during image writing. // Ordered from dirtiest to cleanest (until ArtMethods). enum Bin { + kBinKnownDirty, // Known dirty objects from --dirty-image-objects list kBinMiscDirty, // Dex caches, object locks, etc... kBinClassVerified, // Class verified, but initializers haven't been run // Unknown mix of clean/dirty: @@ -409,8 +399,6 @@ class ImageWriter FINAL { // Verify unwanted classes removed. void CheckNonImageClassesRemoved() REQUIRES_SHARED(Locks::mutator_lock_); - static void CheckNonImageClassesRemovedCallback(mirror::Object* obj, void* arg) - REQUIRES_SHARED(Locks::mutator_lock_); // Lays out where the image objects will be at runtime. void CalculateNewObjectOffsets() @@ -426,18 +414,9 @@ class ImageWriter FINAL { void UnbinObjectsIntoOffset(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_); - static void EnsureBinSlotAssignedCallback(mirror::Object* obj, void* arg) - REQUIRES_SHARED(Locks::mutator_lock_); - static void DeflateMonitorCallback(mirror::Object* obj, void* arg) - REQUIRES_SHARED(Locks::mutator_lock_); - static void UnbinObjectsIntoOffsetCallback(mirror::Object* obj, void* arg) - REQUIRES_SHARED(Locks::mutator_lock_); - // Creates the contiguous image in memory and adjusts pointers. void CopyAndFixupNativeData(size_t oat_index) REQUIRES_SHARED(Locks::mutator_lock_); void CopyAndFixupObjects() REQUIRES_SHARED(Locks::mutator_lock_); - static void CopyAndFixupObjectsCallback(mirror::Object* obj, void* arg) - REQUIRES_SHARED(Locks::mutator_lock_); void CopyAndFixupObject(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_); void CopyAndFixupMethod(ArtMethod* orig, ArtMethod* copy, const ImageInfo& image_info) REQUIRES_SHARED(Locks::mutator_lock_); @@ -496,7 +475,7 @@ class ImageWriter FINAL { // early_exit is true if we had a cyclic dependency anywhere down the chain. bool PruneAppImageClassInternal(ObjPtr<mirror::Class> klass, bool* early_exit, - std::unordered_set<mirror::Class*>* visited) + std::unordered_set<mirror::Object*>* visited) REQUIRES_SHARED(Locks::mutator_lock_); bool IsMultiImage() const { @@ -622,6 +601,9 @@ class ImageWriter FINAL { // Map of dex files to the indexes of oat files that they were compiled into. const std::unordered_map<const DexFile*, size_t>& dex_file_oat_index_map_; + // Set of objects known to be dirty in the image. Can be nullptr if there are none. + const std::unordered_set<std::string>* dirty_image_objects_; + class ComputeLazyFieldsForClassesVisitor; class FixupClassVisitor; class FixupRootVisitor; @@ -633,6 +615,7 @@ class ImageWriter FINAL { class PruneClassLoaderClassesVisitor; class RegisterBootClassPathClassesVisitor; class VisitReferencesVisitor; + class PruneObjectReferenceVisitor; DISALLOW_COPY_AND_ASSIGN(ImageWriter); }; diff --git a/compiler/jit/jit_compiler.cc b/compiler/jit/jit_compiler.cc index 66135414f7..5fdf9ff07c 100644 --- a/compiler/jit/jit_compiler.cc +++ b/compiler/jit/jit_compiler.cc @@ -90,36 +90,20 @@ NO_RETURN static void Usage(const char* fmt, ...) { } JitCompiler::JitCompiler() { - compiler_options_.reset(new CompilerOptions( - CompilerFilter::kDefaultCompilerFilter, - CompilerOptions::kDefaultHugeMethodThreshold, - CompilerOptions::kDefaultLargeMethodThreshold, - CompilerOptions::kDefaultSmallMethodThreshold, - CompilerOptions::kDefaultTinyMethodThreshold, - CompilerOptions::kDefaultNumDexMethodsThreshold, - CompilerOptions::kDefaultInlineMaxCodeUnits, - /* no_inline_from */ nullptr, - CompilerOptions::kDefaultTopKProfileThreshold, - Runtime::Current()->IsJavaDebuggable(), - CompilerOptions::kDefaultGenerateDebugInfo, - /* implicit_null_checks */ true, - /* implicit_so_checks */ true, - /* implicit_suspend_checks */ false, - /* pic */ false, - /* verbose_methods */ nullptr, - /* init_failure_output */ nullptr, - /* abort_on_hard_verifier_failure */ false, - /* dump_cfg_file_name */ "", - /* dump_cfg_append */ false, - /* force_determinism */ false, - RegisterAllocator::kRegisterAllocatorDefault, - /* passes_to_run */ nullptr)); + compiler_options_.reset(new CompilerOptions()); for (const std::string& argument : Runtime::Current()->GetCompilerOptions()) { compiler_options_->ParseCompilerOption(argument, Usage); } // JIT is never PIC, no matter what the runtime compiler options specify. compiler_options_->SetNonPic(); + // Set debuggability based on the runtime value. + compiler_options_->SetDebuggable(Runtime::Current()->IsJavaDebuggable()); + + // Special case max code units for inlining, whose default is "unset" (implictly + // meaning no limit). + compiler_options_->SetInlineMaxCodeUnits(CompilerOptions::kDefaultInlineMaxCodeUnits); + const InstructionSet instruction_set = kRuntimeISA; for (const StringPiece option : Runtime::Current()->GetCompilerOptions()) { VLOG(compiler) << "JIT compiler option " << option; @@ -200,10 +184,8 @@ bool JitCompiler::CompileMethod(Thread* self, ArtMethod* method, bool osr) { { TimingLogger::ScopedTiming t2("Compiling", &logger); JitCodeCache* const code_cache = runtime->GetJit()->GetCodeCache(); - success = compiler_driver_->GetCompiler()->JitCompile(self, code_cache, method, osr); - if (success && (jit_logger_ != nullptr)) { - jit_logger_->WriteLog(code_cache, method, osr); - } + success = compiler_driver_->GetCompiler()->JitCompile( + self, code_cache, method, osr, jit_logger_.get()); } // Trim maps to reduce memory usage. diff --git a/compiler/jit/jit_logger.cc b/compiler/jit/jit_logger.cc index aa4f66773a..2199b64139 100644 --- a/compiler/jit/jit_logger.cc +++ b/compiler/jit/jit_logger.cc @@ -50,11 +50,8 @@ void JitLogger::OpenPerfMapLog() { } } -void JitLogger::WritePerfMapLog(JitCodeCache* code_cache, ArtMethod* method, bool osr) { +void JitLogger::WritePerfMapLog(const void* ptr, size_t code_size, ArtMethod* method) { if (perf_file_ != nullptr) { - const void* ptr = osr ? code_cache->LookupOsrMethodHeader(method)->GetCode() - : method->GetEntryPointFromQuickCompiledCode(); - size_t code_size = code_cache->GetMemorySizeOfCodePointer(ptr); std::string method_name = method->PrettyMethod(); std::ostringstream stream; @@ -270,11 +267,8 @@ void JitLogger::OpenJitDumpLog() { WriteJitDumpHeader(); } -void JitLogger::WriteJitDumpLog(JitCodeCache* code_cache, ArtMethod* method, bool osr) { +void JitLogger::WriteJitDumpLog(const void* ptr, size_t code_size, ArtMethod* method) { if (jit_dump_file_ != nullptr) { - const void* code = osr ? code_cache->LookupOsrMethodHeader(method)->GetCode() - : method->GetEntryPointFromQuickCompiledCode(); - size_t code_size = code_cache->GetMemorySizeOfCodePointer(code); std::string method_name = method->PrettyMethod(); PerfJitCodeLoad jit_code; @@ -285,7 +279,7 @@ void JitLogger::WriteJitDumpLog(JitCodeCache* code_cache, ArtMethod* method, boo jit_code.process_id_ = static_cast<uint32_t>(getpid()); jit_code.thread_id_ = static_cast<uint32_t>(art::GetTid()); jit_code.vma_ = 0x0; - jit_code.code_address_ = reinterpret_cast<uint64_t>(code); + jit_code.code_address_ = reinterpret_cast<uint64_t>(ptr); jit_code.code_size_ = code_size; jit_code.code_id_ = code_index_++; @@ -297,7 +291,7 @@ void JitLogger::WriteJitDumpLog(JitCodeCache* code_cache, ArtMethod* method, boo // Use UNUSED() here to avoid compiler warnings. UNUSED(jit_dump_file_->WriteFully(reinterpret_cast<const char*>(&jit_code), sizeof(jit_code))); UNUSED(jit_dump_file_->WriteFully(method_name.c_str(), method_name.size() + 1)); - UNUSED(jit_dump_file_->WriteFully(code, code_size)); + UNUSED(jit_dump_file_->WriteFully(ptr, code_size)); WriteJitDumpDebugInfo(); } diff --git a/compiler/jit/jit_logger.h b/compiler/jit/jit_logger.h index 460864e8a9..19be9aa88e 100644 --- a/compiler/jit/jit_logger.h +++ b/compiler/jit/jit_logger.h @@ -94,10 +94,10 @@ class JitLogger { OpenJitDumpLog(); } - void WriteLog(JitCodeCache* code_cache, ArtMethod* method, bool osr) + void WriteLog(const void* ptr, size_t code_size, ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) { - WritePerfMapLog(code_cache, method, osr); - WriteJitDumpLog(code_cache, method, osr); + WritePerfMapLog(ptr, code_size, method); + WriteJitDumpLog(ptr, code_size, method); } void CloseLog() { @@ -108,13 +108,13 @@ class JitLogger { private: // For perf-map profiling void OpenPerfMapLog(); - void WritePerfMapLog(JitCodeCache* code_cache, ArtMethod* method, bool osr) + void WritePerfMapLog(const void* ptr, size_t code_size, ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_); void ClosePerfMapLog(); // For perf-inject profiling void OpenJitDumpLog(); - void WriteJitDumpLog(JitCodeCache* code_cache, ArtMethod* method, bool osr) + void WriteJitDumpLog(const void* ptr, size_t code_size, ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_); void CloseJitDumpLog(); diff --git a/compiler/jni/jni_cfi_test.cc b/compiler/jni/jni_cfi_test.cc index 28b7290bef..b552a6e531 100644 --- a/compiler/jni/jni_cfi_test.cc +++ b/compiler/jni/jni_cfi_test.cc @@ -110,20 +110,35 @@ class JNICFITest : public CFITest { } #ifdef ART_ENABLE_CODEGEN_arm +// Run the tests for ARM only with Baker read barriers, as the +// expected generated code contains a Marking Register refresh +// instruction. +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) TEST_ISA(kThumb2) #endif +#endif + #ifdef ART_ENABLE_CODEGEN_arm64 +// Run the tests for ARM64 only with Baker read barriers, as the +// expected generated code contains a Marking Register refresh +// instruction. +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) TEST_ISA(kArm64) #endif +#endif + #ifdef ART_ENABLE_CODEGEN_x86 TEST_ISA(kX86) #endif + #ifdef ART_ENABLE_CODEGEN_x86_64 TEST_ISA(kX86_64) #endif + #ifdef ART_ENABLE_CODEGEN_mips TEST_ISA(kMips) #endif + #ifdef ART_ENABLE_CODEGEN_mips64 TEST_ISA(kMips64) #endif diff --git a/compiler/jni/jni_cfi_test_expected.inc b/compiler/jni/jni_cfi_test_expected.inc index 2710ae9b53..d641fe4251 100644 --- a/compiler/jni/jni_cfi_test_expected.inc +++ b/compiler/jni/jni_cfi_test_expected.inc @@ -1,7 +1,8 @@ static constexpr uint8_t expected_asm_kThumb2[] = { 0x2D, 0xE9, 0xE0, 0x4D, 0x2D, 0xED, 0x10, 0x8A, 0x89, 0xB0, 0x00, 0x90, 0x21, 0x91, 0x8D, 0xED, 0x22, 0x0A, 0x23, 0x92, 0x24, 0x93, 0x88, 0xB0, - 0x08, 0xB0, 0x09, 0xB0, 0xBD, 0xEC, 0x10, 0x8A, 0xBD, 0xE8, 0xE0, 0x8D, + 0x08, 0xB0, 0x09, 0xB0, 0xBD, 0xEC, 0x10, 0x8A, 0xBD, 0xE8, 0xE0, 0x4D, + 0xD9, 0xF8, 0x34, 0x80, 0x70, 0x47, }; static constexpr uint8_t expected_cfi_kThumb2[] = { 0x44, 0x0E, 0x1C, 0x85, 0x07, 0x86, 0x06, 0x87, 0x05, 0x88, 0x04, 0x8A, @@ -13,10 +14,10 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { 0x4E, 0x0E, 0xA0, 0x01, 0x42, 0x0E, 0x80, 0x01, 0x0A, 0x42, 0x0E, 0x5C, 0x44, 0x0E, 0x1C, 0x06, 0x50, 0x06, 0x51, 0x06, 0x52, 0x06, 0x53, 0x06, 0x54, 0x06, 0x55, 0x06, 0x56, 0x06, 0x57, 0x06, 0x58, 0x06, 0x59, 0x06, - 0x5A, 0x06, 0x5B, 0x06, 0x5C, 0x06, 0x5D, 0x06, 0x5E, 0x06, 0x5F, 0x44, + 0x5A, 0x06, 0x5B, 0x06, 0x5C, 0x06, 0x5D, 0x06, 0x5E, 0x06, 0x5F, 0x4A, 0x0B, 0x0E, 0x80, 0x01, }; -// 0x00000000: push {r5, r6, r7, r8, r10, r11, lr} +// 0x00000000: push {r5,r6,r7,r8,r10,r11,lr} // 0x00000004: .cfi_def_cfa_offset: 28 // 0x00000004: .cfi_offset: r5 at cfa-28 // 0x00000004: .cfi_offset: r6 at cfa-24 @@ -25,7 +26,7 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000004: .cfi_offset: r10 at cfa-12 // 0x00000004: .cfi_offset: r11 at cfa-8 // 0x00000004: .cfi_offset: r14 at cfa-4 -// 0x00000004: vpush.f32 {s16-s31} +// 0x00000004: vpush {s16-s31} // 0x00000008: .cfi_def_cfa_offset: 92 // 0x00000008: .cfi_offset_extended: r80 at cfa-92 // 0x00000008: .cfi_offset_extended: r81 at cfa-88 @@ -43,21 +44,21 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000008: .cfi_offset_extended: r93 at cfa-40 // 0x00000008: .cfi_offset_extended: r94 at cfa-36 // 0x00000008: .cfi_offset_extended: r95 at cfa-32 -// 0x00000008: sub sp, sp, #36 +// 0x00000008: sub sp, #36 // 0x0000000a: .cfi_def_cfa_offset: 128 -// 0x0000000a: str r0, [sp, #0] +// 0x0000000a: str r0, [sp] // 0x0000000c: str r1, [sp, #132] -// 0x0000000e: vstr.f32 s0, [sp, #136] +// 0x0000000e: vstr s0, [sp, #136] // 0x00000012: str r2, [sp, #140] // 0x00000014: str r3, [sp, #144] -// 0x00000016: sub sp, sp, #32 +// 0x00000016: sub sp, #32 // 0x00000018: .cfi_def_cfa_offset: 160 -// 0x00000018: add sp, sp, #32 +// 0x00000018: add sp, #32 // 0x0000001a: .cfi_def_cfa_offset: 128 // 0x0000001a: .cfi_remember_state -// 0x0000001a: add sp, sp, #36 +// 0x0000001a: add sp, #36 // 0x0000001c: .cfi_def_cfa_offset: 92 -// 0x0000001c: vpop.f32 {s16-s31} +// 0x0000001c: vpop {s16-s31} // 0x00000020: .cfi_def_cfa_offset: 28 // 0x00000020: .cfi_restore_extended: r80 // 0x00000020: .cfi_restore_extended: r81 @@ -75,9 +76,11 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000020: .cfi_restore_extended: r93 // 0x00000020: .cfi_restore_extended: r94 // 0x00000020: .cfi_restore_extended: r95 -// 0x00000020: pop {r5, r6, r7, r8, r10, r11, pc} -// 0x00000024: .cfi_restore_state -// 0x00000024: .cfi_def_cfa_offset: 128 +// 0x00000020: pop {r5,r6,r7,r8,r10,r11,lr} +// 0x00000024: ldr r8, [tr, #52] ; is_gc_marking +// 0x00000028: bx lr +// 0x0000002a: .cfi_restore_state +// 0x0000002a: .cfi_def_cfa_offset: 128 static constexpr uint8_t expected_asm_kArm64[] = { 0xFF, 0x03, 0x03, 0xD1, 0xF3, 0x53, 0x06, 0xA9, 0xF5, 0x5B, 0x07, 0xA9, @@ -89,7 +92,8 @@ static constexpr uint8_t expected_asm_kArm64[] = { 0xF3, 0x53, 0x46, 0xA9, 0xF5, 0x5B, 0x47, 0xA9, 0xF7, 0x63, 0x48, 0xA9, 0xF9, 0x6B, 0x49, 0xA9, 0xFB, 0x73, 0x4A, 0xA9, 0xFD, 0x7B, 0x4B, 0xA9, 0xE8, 0x27, 0x42, 0x6D, 0xEA, 0x2F, 0x43, 0x6D, 0xEC, 0x37, 0x44, 0x6D, - 0xEE, 0x3F, 0x45, 0x6D, 0xFF, 0x03, 0x03, 0x91, 0xC0, 0x03, 0x5F, 0xD6, + 0xEE, 0x3F, 0x45, 0x6D, 0x74, 0x36, 0x40, 0xB9, 0xFF, 0x03, 0x03, 0x91, + 0xC0, 0x03, 0x5F, 0xD6, }; static constexpr uint8_t expected_cfi_kArm64[] = { 0x44, 0x0E, 0xC0, 0x01, 0x44, 0x93, 0x18, 0x94, 0x16, 0x44, 0x95, 0x14, @@ -101,7 +105,7 @@ static constexpr uint8_t expected_cfi_kArm64[] = { 0xD3, 0xD4, 0x44, 0xD5, 0xD6, 0x44, 0xD7, 0xD8, 0x44, 0xD9, 0xDA, 0x44, 0xDB, 0xDC, 0x44, 0xDD, 0xDE, 0x44, 0x06, 0x48, 0x06, 0x49, 0x44, 0x06, 0x4A, 0x06, 0x4B, 0x44, 0x06, 0x4C, 0x06, 0x4D, 0x44, 0x06, 0x4E, 0x06, - 0x4F, 0x44, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0xC0, 0x01, + 0x4F, 0x48, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0xC0, 0x01, }; // 0x00000000: sub sp, sp, #0xc0 (192) // 0x00000004: .cfi_def_cfa_offset: 192 @@ -175,11 +179,12 @@ static constexpr uint8_t expected_cfi_kArm64[] = { // 0x0000006c: ldp d14, d15, [sp, #80] // 0x00000070: .cfi_restore_extended: r78 // 0x00000070: .cfi_restore_extended: r79 -// 0x00000070: add sp, sp, #0xc0 (192) -// 0x00000074: .cfi_def_cfa_offset: 0 -// 0x00000074: ret -// 0x00000078: .cfi_restore_state -// 0x00000078: .cfi_def_cfa_offset: 192 +// 0x00000070: ldr w20, [tr, #52] ; is_gc_marking +// 0x00000074: add sp, sp, #0xc0 (192) +// 0x00000078: .cfi_def_cfa_offset: 0 +// 0x00000078: ret +// 0x0000007c: .cfi_restore_state +// 0x0000007c: .cfi_def_cfa_offset: 192 static constexpr uint8_t expected_asm_kX86[] = { 0x57, 0x56, 0x55, 0x83, 0xC4, 0xE4, 0x50, 0x89, 0x4C, 0x24, 0x34, 0xF3, diff --git a/compiler/jni/jni_compiler_test.cc b/compiler/jni/jni_compiler_test.cc index b34d9385c8..3460efe474 100644 --- a/compiler/jni/jni_compiler_test.cc +++ b/compiler/jni/jni_compiler_test.cc @@ -32,12 +32,12 @@ #include "mem_map.h" #include "mirror/class-inl.h" #include "mirror/class_loader.h" -#include "mirror/object_array-inl.h" #include "mirror/object-inl.h" +#include "mirror/object_array-inl.h" #include "mirror/stack_trace_element.h" +#include "nativehelper/ScopedLocalRef.h" #include "nativeloader/native_loader.h" #include "runtime.h" -#include "ScopedLocalRef.h" #include "scoped_thread_state_change-inl.h" #include "thread.h" @@ -49,6 +49,9 @@ extern "C" JNIEXPORT jint JNICALL Java_MyClassNatives_sbar(JNIEnv*, jclass, jint return count + 1; } +// TODO: In the Baker read barrier configuration, add checks to ensure +// the Marking Register's value is correct. + namespace art { enum class JniKind { @@ -244,9 +247,9 @@ class JniCompilerTest : public CommonCompilerTest { // Compile the native method before starting the runtime mirror::Class* c = class_linker_->FindClass(soa.Self(), "LMyClassNatives;", loader); const auto pointer_size = class_linker_->GetImagePointerSize(); - ArtMethod* method = direct ? c->FindDirectMethod(method_name, method_sig, pointer_size) : - c->FindVirtualMethod(method_name, method_sig, pointer_size); + ArtMethod* method = c->FindClassMethod(method_name, method_sig, pointer_size); ASSERT_TRUE(method != nullptr) << method_name << " " << method_sig; + ASSERT_EQ(direct, method->IsDirect()) << method_name << " " << method_sig; if (check_generic_jni_) { method->SetEntryPointFromQuickCompiledCode(class_linker_->GetRuntimeQuickGenericJniStub()); } else { diff --git a/compiler/jni/quick/arm64/calling_convention_arm64.cc b/compiler/jni/quick/arm64/calling_convention_arm64.cc index 33f4d77fc2..e086455620 100644 --- a/compiler/jni/quick/arm64/calling_convention_arm64.cc +++ b/compiler/jni/quick/arm64/calling_convention_arm64.cc @@ -108,11 +108,25 @@ static constexpr uint32_t kFpCalleeSpillMask = CalculateFpCalleeSpillMask(); // Calling convention ManagedRegister Arm64ManagedRuntimeCallingConvention::InterproceduralScratchRegister() { - return Arm64ManagedRegister::FromXRegister(X20); // saved on entry restored on exit + // X20 is safe to use as a scratch register: + // - with Baker read barriers, it is reserved as Marking Register, + // and thus does not actually need to be saved/restored; it is + // refreshed on exit (see Arm64JNIMacroAssembler::RemoveFrame); + // - in other cases, it is saved on entry (in + // Arm64JNIMacroAssembler::BuildFrame) and restored on exit (in + // Arm64JNIMacroAssembler::RemoveFrame). + return Arm64ManagedRegister::FromXRegister(X20); } ManagedRegister Arm64JniCallingConvention::InterproceduralScratchRegister() { - return Arm64ManagedRegister::FromXRegister(X20); // saved on entry restored on exit + // X20 is safe to use as a scratch register: + // - with Baker read barriers, it is reserved as Marking Register, + // and thus does not actually need to be saved/restored; it is + // refreshed on exit (see Arm64JNIMacroAssembler::RemoveFrame); + // - in other cases, it is saved on entry (in + // Arm64JNIMacroAssembler::BuildFrame) and restored on exit (in + // Arm64JNIMacroAssembler::RemoveFrame). + return Arm64ManagedRegister::FromXRegister(X20); } static ManagedRegister ReturnRegisterForShorty(const char* shorty) { diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc index c1ac230d43..18ff1c9bb6 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.cc +++ b/compiler/linker/arm/relative_patcher_arm_base.cc @@ -16,6 +16,7 @@ #include "linker/arm/relative_patcher_arm_base.h" +#include "base/stl_util.h" #include "compiled_method.h" #include "linker/output_stream.h" #include "oat.h" diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc index aa5a9457b2..18d6b9ad03 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.cc +++ b/compiler/linker/arm/relative_patcher_thumb2.cc @@ -199,6 +199,24 @@ static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler, // Note: The fake dependency is unnecessary for the slow path. } +// Load the read barrier introspection entrypoint in register `entrypoint` +static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler, + vixl::aarch32::Register entrypoint) { + using vixl::aarch32::MemOperand; + using vixl::aarch32::ip; + // Thread Register. + const vixl::aarch32::Register tr = vixl::aarch32::r9; + + // The register where the read barrier introspection entrypoint is loaded + // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4). + DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister); + // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip.GetCode(), 12u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data) { using namespace vixl::aarch32; // NOLINT(build/namespaces) @@ -233,6 +251,7 @@ void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + raw_ldr_offset; Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); if (width == BakerReadBarrierWidth::kWide) { MemOperand ldr_half_address(lr, ldr_offset + 2); __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12". @@ -278,8 +297,10 @@ void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& MemOperand ldr_address(lr, ldr_offset + 2); __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm", // i.e. Rm+32 because the scale in imm2 is 2. - Register ep_reg(kBakerCcEntrypointRegister); // Insert ip to the entrypoint address to create - __ Bfi(ep_reg, ip, 3, 6); // a switch case target based on the index register. + Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create + // a switch case target based on the index register. __ Mov(ip, base_reg); // Move the base register to ip0. __ Bx(ep_reg); // Jump to the entrypoint's array switch case. break; @@ -309,9 +330,10 @@ void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& " the highest bits and the 'forwarding address' state to have all bits set"); __ Cmp(ip, Operand(0xc0000000)); __ B(hs, &forwarding_address); + Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister // to art_quick_read_barrier_mark_introspection_gc_roots. - Register ep_reg(kBakerCcEntrypointRegister); int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide) ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET; diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index 117684a66b..38c732b8ba 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -59,11 +59,11 @@ inline bool IsAdrpPatch(const LinkerPatch& patch) { case LinkerPatch::Type::kBakerReadBarrierBranch: return false; case LinkerPatch::Type::kMethodRelative: + case LinkerPatch::Type::kMethodBssEntry: case LinkerPatch::Type::kTypeRelative: case LinkerPatch::Type::kTypeBssEntry: case LinkerPatch::Type::kStringRelative: case LinkerPatch::Type::kStringBssEntry: - case LinkerPatch::Type::kDexCacheArray: return patch.LiteralOffset() == patch.PcInsnOffset(); } } @@ -251,20 +251,20 @@ void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, // ADD immediate, 64-bit with imm12 == 0 (unset). if (!kEmitCompilerReadBarrier) { DCHECK(patch.GetType() == LinkerPatch::Type::kMethodRelative || - patch.GetType() == LinkerPatch::Type::kStringRelative || - patch.GetType() == LinkerPatch::Type::kTypeRelative) << patch.GetType(); + patch.GetType() == LinkerPatch::Type::kTypeRelative || + patch.GetType() == LinkerPatch::Type::kStringRelative) << patch.GetType(); } else { // With the read barrier (non-Baker) enabled, it could be kStringBssEntry or kTypeBssEntry. DCHECK(patch.GetType() == LinkerPatch::Type::kMethodRelative || - patch.GetType() == LinkerPatch::Type::kStringRelative || patch.GetType() == LinkerPatch::Type::kTypeRelative || - patch.GetType() == LinkerPatch::Type::kStringBssEntry || - patch.GetType() == LinkerPatch::Type::kTypeBssEntry) << patch.GetType(); + patch.GetType() == LinkerPatch::Type::kStringRelative || + patch.GetType() == LinkerPatch::Type::kTypeBssEntry || + patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); } shift = 0u; // No shift for ADD. } else { // LDR/STR 32-bit or 64-bit with imm12 == 0 (unset). - DCHECK(patch.GetType() == LinkerPatch::Type::kDexCacheArray || + DCHECK(patch.GetType() == LinkerPatch::Type::kMethodBssEntry || patch.GetType() == LinkerPatch::Type::kTypeBssEntry || patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); DCHECK_EQ(insn & 0xbfbffc00, 0xb9000000) << std::hex << insn; @@ -381,6 +381,21 @@ static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler, // Note: The fake dependency is unnecessary for the slow path. } +// Load the read barrier introspection entrypoint in register `entrypoint`. +static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler, + vixl::aarch64::Register entrypoint) { + using vixl::aarch64::MemOperand; + using vixl::aarch64::ip0; + // Thread Register. + const vixl::aarch64::Register tr = vixl::aarch64::x19; + + // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip0.GetCode(), 16u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data) { using namespace vixl::aarch64; // NOLINT(build/namespaces) @@ -412,6 +427,7 @@ void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& a __ Bind(&slow_path); MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset. __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference. // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. @@ -441,6 +457,7 @@ void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& a __ Bind(&slow_path); MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set). __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create // a switch case target based on the index register. @@ -469,6 +486,7 @@ void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& a __ Bind(¬_marked); __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1)); __ B(&forwarding_address, mi); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to // art_quick_read_barrier_mark_introspection_gc_roots. __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET)); diff --git a/compiler/linker/method_bss_mapping_encoder.h b/compiler/linker/method_bss_mapping_encoder.h new file mode 100644 index 0000000000..b2922ec6d2 --- /dev/null +++ b/compiler/linker/method_bss_mapping_encoder.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_METHOD_BSS_MAPPING_ENCODER_H_ +#define ART_COMPILER_LINKER_METHOD_BSS_MAPPING_ENCODER_H_ + +#include "base/enums.h" +#include "base/logging.h" +#include "dex_file.h" +#include "method_bss_mapping.h" + +namespace art { +namespace linker { + +// Helper class for encoding compressed MethodBssMapping. +class MethodBssMappingEncoder { + public: + explicit MethodBssMappingEncoder(PointerSize pointer_size) + : pointer_size_(static_cast<size_t>(pointer_size)) { + entry_.method_index = DexFile::kDexNoIndex16; + entry_.index_mask = 0u; + entry_.bss_offset = static_cast<uint32_t>(-1); + } + + // Try to merge the next method_index -> bss_offset mapping into the current entry. + // Return true on success, false on failure. + bool TryMerge(uint32_t method_index, uint32_t bss_offset) { + DCHECK_NE(method_index, entry_.method_index); + if (entry_.bss_offset + pointer_size_ != bss_offset) { + return false; + } + uint32_t diff = method_index - entry_.method_index; + if (diff > 16u) { + return false; + } + if ((entry_.index_mask & ~(static_cast<uint32_t>(-1) << diff)) != 0u) { + return false; + } + entry_.method_index = method_index; + // Insert the bit indicating the method index we've just overwritten + // and shift bits indicating method indexes before that. + entry_.index_mask = dchecked_integral_cast<uint16_t>( + (static_cast<uint32_t>(entry_.index_mask) | 0x10000u) >> diff); + entry_.bss_offset = bss_offset; + return true; + } + + void Reset(uint32_t method_index, uint32_t bss_offset) { + entry_.method_index = method_index; + entry_.index_mask = 0u; + entry_.bss_offset = bss_offset; + } + + MethodBssMappingEntry GetEntry() { + return entry_; + } + + private: + size_t pointer_size_; + MethodBssMappingEntry entry_; +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_METHOD_BSS_MAPPING_ENCODER_H_ diff --git a/compiler/linker/method_bss_mapping_encoder_test.cc b/compiler/linker/method_bss_mapping_encoder_test.cc new file mode 100644 index 0000000000..1240389bef --- /dev/null +++ b/compiler/linker/method_bss_mapping_encoder_test.cc @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "method_bss_mapping_encoder.h" + +#include "gtest/gtest.h" + +namespace art { +namespace linker { + +TEST(MethodBssMappingEncoder, TryMerge) { + for (PointerSize pointer_size : {PointerSize::k32, PointerSize::k64}) { + size_t raw_pointer_size = static_cast<size_t>(pointer_size); + MethodBssMappingEncoder encoder(pointer_size); + encoder.Reset(1u, 0u); + ASSERT_FALSE(encoder.TryMerge(5u, raw_pointer_size + 1)); // Wrong bss_offset difference. + ASSERT_FALSE(encoder.TryMerge(18u, raw_pointer_size)); // Method index out of range. + ASSERT_TRUE(encoder.TryMerge(5u, raw_pointer_size)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(1u)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(5u)); + ASSERT_FALSE(encoder.GetEntry().CoversIndex(17u)); + ASSERT_FALSE(encoder.TryMerge(17u, 2 * raw_pointer_size + 1)); // Wrong bss_offset difference. + ASSERT_FALSE(encoder.TryMerge(18u, 2 * raw_pointer_size)); // Method index out of range. + ASSERT_TRUE(encoder.TryMerge(17u, 2 * raw_pointer_size)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(1u)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(5u)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(17u)); + ASSERT_EQ(0u, encoder.GetEntry().GetBssOffset(1u, raw_pointer_size)); + ASSERT_EQ(raw_pointer_size, encoder.GetEntry().GetBssOffset(5u, raw_pointer_size)); + ASSERT_EQ(2 * raw_pointer_size, encoder.GetEntry().GetBssOffset(17u, raw_pointer_size)); + ASSERT_EQ(0x0011u, encoder.GetEntry().index_mask); + ASSERT_FALSE(encoder.TryMerge(18u, 2 * raw_pointer_size)); // Method index out of range. + } +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/mips/relative_patcher_mips.cc b/compiler/linker/mips/relative_patcher_mips.cc index 8da530f7cc..3bec30f1e8 100644 --- a/compiler/linker/mips/relative_patcher_mips.cc +++ b/compiler/linker/mips/relative_patcher_mips.cc @@ -49,72 +49,43 @@ void MipsRelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, uint32_t target_offset) { uint32_t anchor_literal_offset = patch.PcInsnOffset(); uint32_t literal_offset = patch.LiteralOffset(); - uint32_t literal_low_offset; - bool dex_cache_array = (patch.GetType() == LinkerPatch::Type::kDexCacheArray); + bool high_patch = ((*code)[literal_offset + 0] == 0x34) && ((*code)[literal_offset + 1] == 0x12); - // Perform basic sanity checks and initialize `literal_low_offset` to point - // to the instruction containing the 16 least significant bits of the - // relative address. - if (is_r6) { - DCHECK_GE(code->size(), 8u); - DCHECK_LE(literal_offset, code->size() - 8u); - DCHECK_EQ(literal_offset, anchor_literal_offset); - // AUIPC reg, offset_high - DCHECK_EQ((*code)[literal_offset + 0], 0x34); - DCHECK_EQ((*code)[literal_offset + 1], 0x12); - DCHECK_EQ(((*code)[literal_offset + 2] & 0x1F), 0x1E); - DCHECK_EQ(((*code)[literal_offset + 3] & 0xFC), 0xEC); - // instr reg(s), offset_low - DCHECK_EQ((*code)[literal_offset + 4], 0x78); - DCHECK_EQ((*code)[literal_offset + 5], 0x56); - literal_low_offset = literal_offset + 4; - } else { - DCHECK_GE(code->size(), 16u); - DCHECK_LE(literal_offset, code->size() - 12u); - DCHECK_GE(literal_offset, 4u); - // The NAL instruction may not precede immediately as the PC+0 value may - // come from HMipsComputeBaseMethodAddress. - if (dex_cache_array) { - DCHECK_EQ(literal_offset + 4u, anchor_literal_offset); - // NAL - DCHECK_EQ((*code)[literal_offset - 4], 0x00); - DCHECK_EQ((*code)[literal_offset - 3], 0x00); - DCHECK_EQ((*code)[literal_offset - 2], 0x10); - DCHECK_EQ((*code)[literal_offset - 1], 0x04); + // Perform basic sanity checks. + if (high_patch) { + if (is_r6) { + // auipc reg, offset_high + DCHECK_EQ(((*code)[literal_offset + 2] & 0x1F), 0x1E); + DCHECK_EQ(((*code)[literal_offset + 3] & 0xFC), 0xEC); + } else { + // lui reg, offset_high + DCHECK_EQ(((*code)[literal_offset + 2] & 0xE0), 0x00); + DCHECK_EQ((*code)[literal_offset + 3], 0x3C); + // addu reg, reg, reg2 + DCHECK_EQ((*code)[literal_offset + 4], 0x21); + DCHECK_EQ(((*code)[literal_offset + 5] & 0x07), 0x00); + DCHECK_EQ(((*code)[literal_offset + 7] & 0xFC), 0x00); } - // LUI reg, offset_high - DCHECK_EQ((*code)[literal_offset + 0], 0x34); - DCHECK_EQ((*code)[literal_offset + 1], 0x12); - DCHECK_EQ(((*code)[literal_offset + 2] & 0xE0), 0x00); - DCHECK_EQ((*code)[literal_offset + 3], 0x3C); - // ADDU reg, reg, reg2 - DCHECK_EQ((*code)[literal_offset + 4], 0x21); - DCHECK_EQ(((*code)[literal_offset + 5] & 0x07), 0x00); - if (dex_cache_array) { - // reg2 is either RA or from HMipsComputeBaseMethodAddress. - DCHECK_EQ(((*code)[literal_offset + 6] & 0x1F), 0x1F); - } - DCHECK_EQ(((*code)[literal_offset + 7] & 0xFC), 0x00); + } else { // instr reg(s), offset_low - DCHECK_EQ((*code)[literal_offset + 8], 0x78); - DCHECK_EQ((*code)[literal_offset + 9], 0x56); - literal_low_offset = literal_offset + 8; + CHECK_EQ((*code)[literal_offset + 0], 0x78); + CHECK_EQ((*code)[literal_offset + 1], 0x56); } // Apply patch. uint32_t anchor_offset = patch_offset - literal_offset + anchor_literal_offset; uint32_t diff = target_offset - anchor_offset; - if (dex_cache_array && !is_r6) { - diff += kDexCacheArrayLwOffset; - } diff += (diff & 0x8000) << 1; // Account for sign extension in "instr reg(s), offset_low". - // LUI reg, offset_high / AUIPC reg, offset_high - (*code)[literal_offset + 0] = static_cast<uint8_t>(diff >> 16); - (*code)[literal_offset + 1] = static_cast<uint8_t>(diff >> 24); - // instr reg(s), offset_low - (*code)[literal_low_offset + 0] = static_cast<uint8_t>(diff >> 0); - (*code)[literal_low_offset + 1] = static_cast<uint8_t>(diff >> 8); + if (high_patch) { + // lui reg, offset_high / auipc reg, offset_high + (*code)[literal_offset + 0] = static_cast<uint8_t>(diff >> 16); + (*code)[literal_offset + 1] = static_cast<uint8_t>(diff >> 24); + } else { + // instr reg(s), offset_low + (*code)[literal_offset + 0] = static_cast<uint8_t>(diff >> 0); + (*code)[literal_offset + 1] = static_cast<uint8_t>(diff >> 8); + } } void MipsRelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* code ATTRIBUTE_UNUSED, diff --git a/compiler/linker/mips/relative_patcher_mips.h b/compiler/linker/mips/relative_patcher_mips.h index 852a345aa6..0b74bd33a4 100644 --- a/compiler/linker/mips/relative_patcher_mips.h +++ b/compiler/linker/mips/relative_patcher_mips.h @@ -46,9 +46,6 @@ class MipsRelativePatcher FINAL : public RelativePatcher { uint32_t patch_offset) OVERRIDE; private: - // We'll maximize the range of a single load instruction for dex cache array accesses - // by aligning offset -32768 with the offset of the first used element. - static constexpr uint32_t kDexCacheArrayLwOffset = 0x8000; bool is_r6; DISALLOW_COPY_AND_ASSIGN(MipsRelativePatcher); diff --git a/compiler/linker/mips/relative_patcher_mips32r6_test.cc b/compiler/linker/mips/relative_patcher_mips32r6_test.cc index 63ad8a58d5..d1a75e28a2 100644 --- a/compiler/linker/mips/relative_patcher_mips32r6_test.cc +++ b/compiler/linker/mips/relative_patcher_mips32r6_test.cc @@ -26,7 +26,9 @@ class Mips32r6RelativePatcherTest : public RelativePatcherTest { protected: static const uint8_t kUnpatchedPcRelativeRawCode[]; - static const uint32_t kLiteralOffset; + static const uint32_t kLiteralOffsetHigh; + static const uint32_t kLiteralOffsetLow1; + static const uint32_t kLiteralOffsetLow2; static const uint32_t kAnchorOffset; static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode; @@ -44,9 +46,11 @@ class Mips32r6RelativePatcherTest : public RelativePatcherTest { const uint8_t Mips32r6RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = { 0x34, 0x12, 0x5E, 0xEE, // auipc s2, high(diff); placeholder = 0x1234 0x78, 0x56, 0x52, 0x26, // addiu s2, s2, low(diff); placeholder = 0x5678 + 0x78, 0x56, 0x52, 0x8E, // lw s2, (low(diff))(s2) ; placeholder = 0x5678 }; -const uint32_t Mips32r6RelativePatcherTest::kLiteralOffset = 0; // At auipc (where - // patching starts). +const uint32_t Mips32r6RelativePatcherTest::kLiteralOffsetHigh = 0; // At auipc. +const uint32_t Mips32r6RelativePatcherTest::kLiteralOffsetLow1 = 4; // At addiu. +const uint32_t Mips32r6RelativePatcherTest::kLiteralOffsetLow2 = 8; // At lw. const uint32_t Mips32r6RelativePatcherTest::kAnchorOffset = 0; // At auipc (where PC+0 points). const ArrayRef<const uint8_t> Mips32r6RelativePatcherTest::kUnpatchedPcRelativeCode( kUnpatchedPcRelativeRawCode); @@ -60,11 +64,12 @@ void Mips32r6RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const Link ASSERT_TRUE(result.first); uint32_t diff = target_offset - (result.second + kAnchorOffset); - diff += (diff & 0x8000) << 1; // Account for sign extension in addiu. + diff += (diff & 0x8000) << 1; // Account for sign extension in addiu/lw. const uint8_t expected_code[] = { static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24), 0x5E, 0xEE, static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), 0x52, 0x26, + static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), 0x52, 0x8E, }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } @@ -75,7 +80,9 @@ void Mips32r6RelativePatcherTest::TestStringBssEntry(uint32_t bss_begin, string_index_to_offset_map_.Put(kStringIndex, string_entry_offset); bss_begin_ = bss_begin; LinkerPatch patches[] = { - LinkerPatch::StringBssEntryPatch(kLiteralOffset, nullptr, kAnchorOffset, kStringIndex) + LinkerPatch::StringBssEntryPatch(kLiteralOffsetHigh, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::StringBssEntryPatch(kLiteralOffsetLow1, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::StringBssEntryPatch(kLiteralOffsetLow2, nullptr, kAnchorOffset, kStringIndex) }; CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), bss_begin_ + string_entry_offset); } @@ -84,7 +91,9 @@ void Mips32r6RelativePatcherTest::TestStringReference(uint32_t string_offset) { constexpr uint32_t kStringIndex = 1u; string_index_to_offset_map_.Put(kStringIndex, string_offset); LinkerPatch patches[] = { - LinkerPatch::RelativeStringPatch(kLiteralOffset, nullptr, kAnchorOffset, kStringIndex) + LinkerPatch::RelativeStringPatch(kLiteralOffsetHigh, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::RelativeStringPatch(kLiteralOffsetLow1, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::RelativeStringPatch(kLiteralOffsetLow2, nullptr, kAnchorOffset, kStringIndex) }; CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset); } diff --git a/compiler/linker/mips/relative_patcher_mips_test.cc b/compiler/linker/mips/relative_patcher_mips_test.cc index 961b31266f..2f7a0752a6 100644 --- a/compiler/linker/mips/relative_patcher_mips_test.cc +++ b/compiler/linker/mips/relative_patcher_mips_test.cc @@ -26,7 +26,9 @@ class MipsRelativePatcherTest : public RelativePatcherTest { protected: static const uint8_t kUnpatchedPcRelativeRawCode[]; - static const uint32_t kLiteralOffset; + static const uint32_t kLiteralOffsetHigh; + static const uint32_t kLiteralOffsetLow1; + static const uint32_t kLiteralOffsetLow2; static const uint32_t kAnchorOffset; static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode; @@ -46,8 +48,11 @@ const uint8_t MipsRelativePatcherTest::kUnpatchedPcRelativeRawCode[] = { 0x34, 0x12, 0x12, 0x3C, // lui s2, high(diff); placeholder = 0x1234 0x21, 0x90, 0x5F, 0x02, // addu s2, s2, ra 0x78, 0x56, 0x52, 0x26, // addiu s2, s2, low(diff); placeholder = 0x5678 + 0x78, 0x56, 0x52, 0x8E, // lw s2, (low(diff))(s2) ; placeholder = 0x5678 }; -const uint32_t MipsRelativePatcherTest::kLiteralOffset = 4; // At lui (where patching starts). +const uint32_t MipsRelativePatcherTest::kLiteralOffsetHigh = 4; // At lui. +const uint32_t MipsRelativePatcherTest::kLiteralOffsetLow1 = 12; // At addiu. +const uint32_t MipsRelativePatcherTest::kLiteralOffsetLow2 = 16; // At lw. const uint32_t MipsRelativePatcherTest::kAnchorOffset = 8; // At addu (where PC+0 points). const ArrayRef<const uint8_t> MipsRelativePatcherTest::kUnpatchedPcRelativeCode( kUnpatchedPcRelativeRawCode); @@ -61,14 +66,14 @@ void MipsRelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPa ASSERT_TRUE(result.first); uint32_t diff = target_offset - (result.second + kAnchorOffset); - CHECK_NE(patches[0].GetType(), LinkerPatch::Type::kDexCacheArray); - diff += (diff & 0x8000) << 1; // Account for sign extension in addiu. + diff += (diff & 0x8000) << 1; // Account for sign extension in addiu/lw. const uint8_t expected_code[] = { 0x00, 0x00, 0x10, 0x04, static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24), 0x12, 0x3C, 0x21, 0x90, 0x5F, 0x02, static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), 0x52, 0x26, + static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), 0x52, 0x8E, }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } @@ -79,7 +84,9 @@ void MipsRelativePatcherTest::TestStringBssEntry(uint32_t bss_begin, string_index_to_offset_map_.Put(kStringIndex, string_entry_offset); bss_begin_ = bss_begin; LinkerPatch patches[] = { - LinkerPatch::StringBssEntryPatch(kLiteralOffset, nullptr, kAnchorOffset, kStringIndex) + LinkerPatch::StringBssEntryPatch(kLiteralOffsetHigh, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::StringBssEntryPatch(kLiteralOffsetLow1, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::StringBssEntryPatch(kLiteralOffsetLow2, nullptr, kAnchorOffset, kStringIndex) }; CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), bss_begin_ + string_entry_offset); } @@ -88,7 +95,9 @@ void MipsRelativePatcherTest::TestStringReference(uint32_t string_offset) { constexpr uint32_t kStringIndex = 1u; string_index_to_offset_map_.Put(kStringIndex, string_offset); LinkerPatch patches[] = { - LinkerPatch::RelativeStringPatch(kLiteralOffset, nullptr, kAnchorOffset, kStringIndex) + LinkerPatch::RelativeStringPatch(kLiteralOffsetHigh, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::RelativeStringPatch(kLiteralOffsetLow1, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::RelativeStringPatch(kLiteralOffsetLow2, nullptr, kAnchorOffset, kStringIndex) }; CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset); } diff --git a/compiler/linker/mips64/relative_patcher_mips64.cc b/compiler/linker/mips64/relative_patcher_mips64.cc index 3488d6d21c..d9f4758eb1 100644 --- a/compiler/linker/mips64/relative_patcher_mips64.cc +++ b/compiler/linker/mips64/relative_patcher_mips64.cc @@ -36,38 +36,11 @@ uint32_t Mips64RelativePatcher::WriteThunks(OutputStream* out ATTRIBUTE_UNUSED, return offset; // No thunks added; no limit on relative call distance. } -void Mips64RelativePatcher::PatchCall(std::vector<uint8_t>* code, - uint32_t literal_offset, - uint32_t patch_offset, - uint32_t target_offset) { - // Basic sanity checks. - DCHECK_GE(code->size(), 8u); - DCHECK_LE(literal_offset, code->size() - 8u); - // auipc reg, offset_high - DCHECK_EQ((*code)[literal_offset + 0], 0x34); - DCHECK_EQ((*code)[literal_offset + 1], 0x12); - DCHECK_EQ(((*code)[literal_offset + 2] & 0x1F), 0x1E); - DCHECK_EQ(((*code)[literal_offset + 3] & 0xFC), 0xEC); - // jialc reg, offset_low - DCHECK_EQ((*code)[literal_offset + 4], 0x78); - DCHECK_EQ((*code)[literal_offset + 5], 0x56); - DCHECK_EQ(((*code)[literal_offset + 6] & 0xE0), 0x00); - DCHECK_EQ((*code)[literal_offset + 7], 0xF8); - - // Apply patch. - uint32_t diff = target_offset - patch_offset; - // Note that a combination of auipc with an instruction that adds a sign-extended - // 16-bit immediate operand (e.g. jialc) provides a PC-relative range of - // PC-0x80000000 to PC+0x7FFF7FFF on MIPS64, that is, short of 2GB on one end - // by 32KB. - diff += (diff & 0x8000) << 1; // Account for sign extension in jialc. - - // auipc reg, offset_high - (*code)[literal_offset + 0] = static_cast<uint8_t>(diff >> 16); - (*code)[literal_offset + 1] = static_cast<uint8_t>(diff >> 24); - // jialc reg, offset_low - (*code)[literal_offset + 4] = static_cast<uint8_t>(diff >> 0); - (*code)[literal_offset + 5] = static_cast<uint8_t>(diff >> 8); +void Mips64RelativePatcher::PatchCall(std::vector<uint8_t>* code ATTRIBUTE_UNUSED, + uint32_t literal_offset ATTRIBUTE_UNUSED, + uint32_t patch_offset ATTRIBUTE_UNUSED, + uint32_t target_offset ATTRIBUTE_UNUSED) { + UNIMPLEMENTED(FATAL) << "PatchCall unimplemented on MIPS64"; } void Mips64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, @@ -76,19 +49,18 @@ void Mips64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, uint32_t target_offset) { uint32_t anchor_literal_offset = patch.PcInsnOffset(); uint32_t literal_offset = patch.LiteralOffset(); + bool high_patch = ((*code)[literal_offset + 0] == 0x34) && ((*code)[literal_offset + 1] == 0x12); - // Basic sanity checks. - DCHECK_GE(code->size(), 8u); - DCHECK_LE(literal_offset, code->size() - 8u); - DCHECK_EQ(literal_offset, anchor_literal_offset); - // auipc reg, offset_high - DCHECK_EQ((*code)[literal_offset + 0], 0x34); - DCHECK_EQ((*code)[literal_offset + 1], 0x12); - DCHECK_EQ(((*code)[literal_offset + 2] & 0x1F), 0x1E); - DCHECK_EQ(((*code)[literal_offset + 3] & 0xFC), 0xEC); - // instr reg(s), offset_low - DCHECK_EQ((*code)[literal_offset + 4], 0x78); - DCHECK_EQ((*code)[literal_offset + 5], 0x56); + // Perform basic sanity checks. + if (high_patch) { + // auipc reg, offset_high + DCHECK_EQ(((*code)[literal_offset + 2] & 0x1F), 0x1E); + DCHECK_EQ(((*code)[literal_offset + 3] & 0xFC), 0xEC); + } else { + // instr reg(s), offset_low + CHECK_EQ((*code)[literal_offset + 0], 0x78); + CHECK_EQ((*code)[literal_offset + 1], 0x56); + } // Apply patch. uint32_t anchor_offset = patch_offset - literal_offset + anchor_literal_offset; @@ -97,14 +69,17 @@ void Mips64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, // 16-bit immediate operand (e.g. ld) provides a PC-relative range of // PC-0x80000000 to PC+0x7FFF7FFF on MIPS64, that is, short of 2GB on one end // by 32KB. - diff += (diff & 0x8000) << 1; // Account for sign extension in instruction following auipc. + diff += (diff & 0x8000) << 1; // Account for sign extension in "instr reg(s), offset_low". - // auipc reg, offset_high - (*code)[literal_offset + 0] = static_cast<uint8_t>(diff >> 16); - (*code)[literal_offset + 1] = static_cast<uint8_t>(diff >> 24); - // instr reg(s), offset_low - (*code)[literal_offset + 4] = static_cast<uint8_t>(diff >> 0); - (*code)[literal_offset + 5] = static_cast<uint8_t>(diff >> 8); + if (high_patch) { + // auipc reg, offset_high + (*code)[literal_offset + 0] = static_cast<uint8_t>(diff >> 16); + (*code)[literal_offset + 1] = static_cast<uint8_t>(diff >> 24); + } else { + // instr reg(s), offset_low + (*code)[literal_offset + 0] = static_cast<uint8_t>(diff >> 0); + (*code)[literal_offset + 1] = static_cast<uint8_t>(diff >> 8); + } } void Mips64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* code ATTRIBUTE_UNUSED, diff --git a/compiler/linker/mips64/relative_patcher_mips64_test.cc b/compiler/linker/mips64/relative_patcher_mips64_test.cc index 9c9e24a96e..a5f494d645 100644 --- a/compiler/linker/mips64/relative_patcher_mips64_test.cc +++ b/compiler/linker/mips64/relative_patcher_mips64_test.cc @@ -27,10 +27,11 @@ class Mips64RelativePatcherTest : public RelativePatcherTest { protected: static const uint8_t kUnpatchedPcRelativeRawCode[]; static const uint8_t kUnpatchedPcRelativeCallRawCode[]; - static const uint32_t kLiteralOffset; + static const uint32_t kLiteralOffsetHigh; + static const uint32_t kLiteralOffsetLow1; + static const uint32_t kLiteralOffsetLow2; static const uint32_t kAnchorOffset; static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode; - static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCallCode; uint32_t GetMethodOffset(uint32_t method_idx) { auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx)); @@ -44,19 +45,16 @@ class Mips64RelativePatcherTest : public RelativePatcherTest { }; const uint8_t Mips64RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = { - 0x34, 0x12, 0x5E, 0xEE, // auipc s2, high(diff); placeholder = 0x1234 + 0x34, 0x12, 0x5E, 0xEE, // auipc s2, high(diff); placeholder = 0x1234 0x78, 0x56, 0x52, 0x66, // daddiu s2, s2, low(diff); placeholder = 0x5678 + 0x78, 0x56, 0x52, 0x9E, // lwu s2, (low(diff))(s2) ; placeholder = 0x5678 }; -const uint8_t Mips64RelativePatcherTest::kUnpatchedPcRelativeCallRawCode[] = { - 0x34, 0x12, 0x3E, 0xEC, // auipc at, high(diff); placeholder = 0x1234 - 0x78, 0x56, 0x01, 0xF8, // jialc at, low(diff); placeholder = 0x5678 -}; -const uint32_t Mips64RelativePatcherTest::kLiteralOffset = 0; // At auipc (where patching starts). +const uint32_t Mips64RelativePatcherTest::kLiteralOffsetHigh = 0; // At auipc. +const uint32_t Mips64RelativePatcherTest::kLiteralOffsetLow1 = 4; // At daddiu. +const uint32_t Mips64RelativePatcherTest::kLiteralOffsetLow2 = 8; // At lwu. const uint32_t Mips64RelativePatcherTest::kAnchorOffset = 0; // At auipc (where PC+0 points). const ArrayRef<const uint8_t> Mips64RelativePatcherTest::kUnpatchedPcRelativeCode( kUnpatchedPcRelativeRawCode); -const ArrayRef<const uint8_t> Mips64RelativePatcherTest::kUnpatchedPcRelativeCallCode( - kUnpatchedPcRelativeCallRawCode); void Mips64RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset) { @@ -67,11 +65,12 @@ void Mips64RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const Linker ASSERT_TRUE(result.first); uint32_t diff = target_offset - (result.second + kAnchorOffset); - diff += (diff & 0x8000) << 1; // Account for sign extension in instruction following auipc. + diff += (diff & 0x8000) << 1; // Account for sign extension in daddiu/lwu. const uint8_t expected_code[] = { static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24), 0x5E, 0xEE, static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), 0x52, 0x66, + static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), 0x52, 0x9E, }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } @@ -82,7 +81,9 @@ void Mips64RelativePatcherTest::TestStringBssEntry(uint32_t bss_begin, string_index_to_offset_map_.Put(kStringIndex, string_entry_offset); bss_begin_ = bss_begin; LinkerPatch patches[] = { - LinkerPatch::StringBssEntryPatch(kLiteralOffset, nullptr, kAnchorOffset, kStringIndex) + LinkerPatch::StringBssEntryPatch(kLiteralOffsetHigh, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::StringBssEntryPatch(kLiteralOffsetLow1, nullptr, kAnchorOffset, kStringIndex), + LinkerPatch::StringBssEntryPatch(kLiteralOffsetLow2, nullptr, kAnchorOffset, kStringIndex) }; CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), bss_begin_ + string_entry_offset); } @@ -91,38 +92,5 @@ TEST_F(Mips64RelativePatcherTest, StringBssEntry) { TestStringBssEntry(/* bss_begin */ 0x12345678, /* string_entry_offset */ 0x1234); } -TEST_F(Mips64RelativePatcherTest, CallOther) { - LinkerPatch method1_patches[] = { - LinkerPatch::RelativeCodePatch(kLiteralOffset, nullptr, 2u), - }; - AddCompiledMethod(MethodRef(1u), - kUnpatchedPcRelativeCallCode, - ArrayRef<const LinkerPatch>(method1_patches)); - LinkerPatch method2_patches[] = { - LinkerPatch::RelativeCodePatch(kLiteralOffset, nullptr, 1u), - }; - AddCompiledMethod(MethodRef(2u), - kUnpatchedPcRelativeCallCode, - ArrayRef<const LinkerPatch>(method2_patches)); - Link(); - - uint32_t method1_offset = GetMethodOffset(1u); - uint32_t method2_offset = GetMethodOffset(2u); - uint32_t diff_after = method2_offset - (method1_offset + kAnchorOffset /* PC adjustment */); - diff_after += (diff_after & 0x8000) << 1; // Account for sign extension in jialc. - static const uint8_t method1_expected_code[] = { - static_cast<uint8_t>(diff_after >> 16), static_cast<uint8_t>(diff_after >> 24), 0x3E, 0xEC, - static_cast<uint8_t>(diff_after), static_cast<uint8_t>(diff_after >> 8), 0x01, 0xF8, - }; - EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code))); - uint32_t diff_before = method1_offset - (method2_offset + kAnchorOffset /* PC adjustment */); - diff_before += (diff_before & 0x8000) << 1; // Account for sign extension in jialc. - static const uint8_t method2_expected_code[] = { - static_cast<uint8_t>(diff_before >> 16), static_cast<uint8_t>(diff_before >> 24), 0x3E, 0xEC, - static_cast<uint8_t>(diff_before), static_cast<uint8_t>(diff_before >> 8), 0x01, 0xF8, - }; - EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code))); -} - } // namespace linker } // namespace art diff --git a/compiler/linker/output_stream_test.cc b/compiler/linker/output_stream_test.cc index 84c76f2c6c..09fef29d48 100644 --- a/compiler/linker/output_stream_test.cc +++ b/compiler/linker/output_stream_test.cc @@ -19,7 +19,6 @@ #include "base/unix_file/fd_file.h" #include "base/logging.h" -#include "base/stl_util.h" #include "buffered_output_stream.h" #include "common_runtime_test.h" @@ -79,7 +78,7 @@ TEST_F(OutputStreamTest, File) { TEST_F(OutputStreamTest, Buffered) { ScratchFile tmp; { - BufferedOutputStream buffered_output_stream(MakeUnique<FileOutputStream>(tmp.GetFile())); + BufferedOutputStream buffered_output_stream(std::make_unique<FileOutputStream>(tmp.GetFile())); SetOutputStream(buffered_output_stream); GenerateTestOutput(); } @@ -125,7 +124,7 @@ TEST_F(OutputStreamTest, BufferedFlush) { bool flush_called; }; - std::unique_ptr<CheckingOutputStream> cos = MakeUnique<CheckingOutputStream>(); + std::unique_ptr<CheckingOutputStream> cos = std::make_unique<CheckingOutputStream>(); CheckingOutputStream* checking_output_stream = cos.get(); BufferedOutputStream buffered(std::move(cos)); ASSERT_FALSE(checking_output_stream->flush_called); diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index 1578c0cd3e..6f8904979d 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -19,6 +19,7 @@ #include "arch/instruction_set_features.h" #include "art_method-inl.h" #include "base/enums.h" +#include "base/stl_util.h" #include "base/unix_file/fd_file.h" #include "class_linker.h" #include "common_compiler_test.h" @@ -103,8 +104,8 @@ class OatTest : public CommonCompilerTest { compiler_options_->ParseCompilerOption(option, Usage); } verification_results_.reset(new VerificationResults(compiler_options_.get())); - callbacks_.reset(new QuickCompilerCallbacks(verification_results_.get(), - CompilerCallbacks::CallbackMode::kCompileApp)); + callbacks_.reset(new QuickCompilerCallbacks(CompilerCallbacks::CallbackMode::kCompileApp)); + callbacks_->SetVerificationResults(verification_results_.get()); Runtime::Current()->SetCompilerCallbacks(callbacks_.get()); timer_.reset(new CumulativeLogger("Compilation times")); compiler_driver_.reset(new CompilerDriver(compiler_options_.get(), @@ -220,11 +221,12 @@ class OatTest : public CommonCompilerTest { elf_writer->PrepareDynamicSection(rodata_size, text_size, oat_writer.GetBssSize(), + oat_writer.GetBssMethodsOffset(), oat_writer.GetBssRootsOffset()); if (kIsVdexEnabled) { - std::unique_ptr<BufferedOutputStream> vdex_out( - MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(vdex_file))); + std::unique_ptr<BufferedOutputStream> vdex_out = + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file)); if (!oat_writer.WriteVerifierDeps(vdex_out.get(), nullptr)) { return false; } @@ -398,8 +400,7 @@ TEST_F(OatTest, WriteRead) { if (kCompile) { TimingLogger timings2("OatTest::WriteRead", false, false); compiler_driver_->SetDexFilesForOatFile(class_linker->GetBootClassPath()); - compiler_driver_->CompileAll( - class_loader, class_linker->GetBootClassPath(), /* verifier_deps */ nullptr, &timings2); + compiler_driver_->CompileAll(class_loader, class_linker->GetBootClassPath(), &timings2); } ScratchFile tmp_oat, tmp_vdex(tmp_oat, ".vdex"); @@ -413,8 +414,7 @@ TEST_F(OatTest, WriteRead) { ASSERT_TRUE(success); if (kCompile) { // OatWriter strips the code, regenerate to compare - compiler_driver_->CompileAll( - class_loader, class_linker->GetBootClassPath(), /* verifier_deps */ nullptr, &timings); + compiler_driver_->CompileAll(class_loader, class_linker->GetBootClassPath(), &timings); } std::unique_ptr<OatFile> oat_file(OatFile::Open(tmp_oat.GetFilename(), tmp_oat.GetFilename(), @@ -483,7 +483,7 @@ TEST_F(OatTest, WriteRead) { TEST_F(OatTest, OatHeaderSizeCheck) { // If this test is failing and you have to update these constants, // it is time to update OatHeader::kOatVersion - EXPECT_EQ(72U, sizeof(OatHeader)); + EXPECT_EQ(76U, sizeof(OatHeader)); EXPECT_EQ(4U, sizeof(OatMethodOffsets)); EXPECT_EQ(24U, sizeof(OatQuickMethodHeader)); EXPECT_EQ(161 * static_cast<size_t>(GetInstructionSetPointerSize(kRuntimeISA)), @@ -538,7 +538,7 @@ TEST_F(OatTest, EmptyTextSection) { soa.Decode<mirror::ClassLoader>(class_loader).Ptr()); } compiler_driver_->SetDexFilesForOatFile(dex_files); - compiler_driver_->CompileAll(class_loader, dex_files, /* verifier_deps */ nullptr, &timings); + compiler_driver_->CompileAll(class_loader, dex_files, &timings); ScratchFile tmp_oat, tmp_vdex(tmp_oat, ".vdex"); SafeMap<std::string, std::string> key_value_store; diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index fed2d34cdb..0b2d7f423b 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -22,7 +22,7 @@ #include "arch/arm64/instruction_set_features_arm64.h" #include "art_method-inl.h" #include "base/allocator.h" -#include "base/bit_vector.h" +#include "base/bit_vector-inl.h" #include "base/enums.h" #include "base/file_magic.h" #include "base/stl_util.h" @@ -41,6 +41,7 @@ #include "image_writer.h" #include "linker/buffered_output_stream.h" #include "linker/file_output_stream.h" +#include "linker/method_bss_mapping_encoder.h" #include "linker/multi_oat_relative_patcher.h" #include "linker/output_stream.h" #include "mirror/array.h" @@ -61,6 +62,9 @@ namespace art { namespace { // anonymous namespace +// If we write dex layout info in the oat file. +static constexpr bool kWriteDexLayoutInfo = true; + typedef DexFile::Header __attribute__((aligned(1))) UnalignedDexFileHeader; const UnalignedDexFileHeader* AsUnalignedDexFileHeader(const uint8_t* raw_data) { @@ -157,26 +161,61 @@ class OatWriter::DexFileSource { const void* source_; }; +// OatClassHeader is the header only part of the oat class that is required even when compilation +// is not enabled. +class OatWriter::OatClassHeader { + public: + OatClassHeader(uint32_t offset, + uint32_t num_non_null_compiled_methods, + uint32_t num_methods, + mirror::Class::Status status) + : status_(status), + offset_(offset) { + // We just arbitrarily say that 0 methods means kOatClassNoneCompiled and that we won't use + // kOatClassAllCompiled unless there is at least one compiled method. This means in an + // interpreter only system, we can assert that all classes are kOatClassNoneCompiled. + if (num_non_null_compiled_methods == 0) { + type_ = kOatClassNoneCompiled; + } else if (num_non_null_compiled_methods == num_methods) { + type_ = kOatClassAllCompiled; + } else { + type_ = kOatClassSomeCompiled; + } + } + + bool Write(OatWriter* oat_writer, OutputStream* out, const size_t file_offset) const; + + static size_t SizeOf() { + return sizeof(status_) + sizeof(type_); + } + + // Data to write. + static_assert(mirror::Class::Status::kStatusMax < (1 << 16), "class status won't fit in 16bits"); + int16_t status_; + + static_assert(OatClassType::kOatClassMax < (1 << 16), "oat_class type won't fit in 16bits"); + uint16_t type_; + + // Offset of start of OatClass from beginning of OatHeader. It is + // used to validate file position when writing. + uint32_t offset_; +}; + +// The actual oat class body contains the information about compiled methods. It is only required +// for compiler filters that have any compilation. class OatWriter::OatClass { public: - OatClass(size_t offset, - const dchecked_vector<CompiledMethod*>& compiled_methods, - uint32_t num_non_null_compiled_methods, - mirror::Class::Status status); + OatClass(const dchecked_vector<CompiledMethod*>& compiled_methods, + uint32_t compiled_methods_with_code, + uint16_t oat_class_type); OatClass(OatClass&& src) = default; - size_t GetOatMethodOffsetsOffsetFromOatHeader(size_t class_def_method_index_) const; - size_t GetOatMethodOffsetsOffsetFromOatClass(size_t class_def_method_index_) const; size_t SizeOf() const; - bool Write(OatWriter* oat_writer, OutputStream* out, const size_t file_offset) const; + bool Write(OatWriter* oat_writer, OutputStream* out) const; CompiledMethod* GetCompiledMethod(size_t class_def_method_index) const { return compiled_methods_[class_def_method_index]; } - // Offset of start of OatClass from beginning of OatHeader. It is - // used to validate file position when writing. - size_t offset_; - // CompiledMethods for each class_def_method_index, or null if no method is available. dchecked_vector<CompiledMethod*> compiled_methods_; @@ -187,13 +226,6 @@ class OatWriter::OatClass { dchecked_vector<uint32_t> oat_method_offsets_offsets_from_oat_class_; // Data to write. - - static_assert(mirror::Class::Status::kStatusMax < (1 << 16), "class status won't fit in 16bits"); - int16_t status_; - - static_assert(OatClassType::kOatClassMax < (1 << 16), "oat_class type won't fit in 16bits"); - uint16_t type_; - uint32_t method_bitmap_size_; // bit vector indexed by ClassDef method index. When @@ -230,12 +262,14 @@ class OatWriter::OatDexFile { return dex_file_location_data_; } - void ReserveClassOffsets(OatWriter* oat_writer); - size_t SizeOf() const; bool Write(OatWriter* oat_writer, OutputStream* out) const; bool WriteClassOffsets(OatWriter* oat_writer, OutputStream* out); + size_t GetClassOffsetsRawSize() const { + return class_offsets_.size() * sizeof(class_offsets_[0]); + } + // The source of the dex file. DexFileSource source_; @@ -256,15 +290,16 @@ class OatWriter::OatDexFile { uint32_t dex_file_offset_; uint32_t class_offsets_offset_; uint32_t lookup_table_offset_; + uint32_t method_bss_mapping_offset_; + uint32_t dex_sections_layout_offset_; // Data to write to a separate section. dchecked_vector<uint32_t> class_offsets_; - private: - size_t GetClassOffsetsRawSize() const { - return class_offsets_.size() * sizeof(class_offsets_[0]); - } + // Dex section layout info to serialize. + DexLayoutSections dex_sections_layout_; + private: DISALLOW_COPY_AND_ASSIGN(OatDexFile); }; @@ -294,7 +329,10 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCo oat_size_(0u), bss_start_(0u), bss_size_(0u), + bss_methods_offset_(0u), bss_roots_offset_(0u), + bss_method_entry_references_(), + bss_method_entries_(), bss_type_entries_(), bss_string_entries_(), oat_data_offset_(0u), @@ -331,6 +369,10 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCo size_oat_dex_file_offset_(0), size_oat_dex_file_class_offsets_offset_(0), size_oat_dex_file_lookup_table_offset_(0), + size_oat_dex_file_dex_layout_sections_offset_(0), + size_oat_dex_file_dex_layout_sections_(0), + size_oat_dex_file_dex_layout_sections_alignment_(0), + size_oat_dex_file_method_bss_mapping_offset_(0), size_oat_lookup_table_alignment_(0), size_oat_lookup_table_(0), size_oat_class_offsets_alignment_(0), @@ -339,6 +381,7 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCo size_oat_class_status_(0), size_oat_class_method_bitmaps_(0), size_oat_class_method_offsets_(0), + size_method_bss_mappings_(0u), relative_patcher_(nullptr), absolute_patch_locations_(), profile_compilation_info_(info) { @@ -468,8 +511,8 @@ bool OatWriter::AddRawDexFileSource(const ArrayRef<const uint8_t>& data, return true; } -dchecked_vector<const char*> OatWriter::GetSourceLocations() const { - dchecked_vector<const char*> locations; +dchecked_vector<std::string> OatWriter::GetSourceLocations() const { + dchecked_vector<std::string> locations; locations.reserve(oat_dex_files_.size()); for (const OatDexFile& oat_dex_file : oat_dex_files_) { locations.push_back(oat_dex_file.GetLocation()); @@ -477,6 +520,11 @@ dchecked_vector<const char*> OatWriter::GetSourceLocations() const { return locations; } +bool OatWriter::MayHaveCompiledMethods() const { + return CompilerFilter::IsAnyCompilationEnabled( + GetCompilerDriver()->GetCompilerOptions().GetCompilerFilter()); +} + bool OatWriter::WriteAndOpenDexFiles( File* vdex_file, OutputStream* oat_rodata, @@ -502,17 +550,16 @@ bool OatWriter::WriteAndOpenDexFiles( // Reserve space for Vdex header and checksums. vdex_size_ = sizeof(VdexFile::Header) + oat_dex_files_.size() * sizeof(VdexFile::VdexChecksum); } - size_t oat_data_offset = InitOatHeader(instruction_set, - instruction_set_features, - dchecked_integral_cast<uint32_t>(oat_dex_files_.size()), - key_value_store); - oat_size_ = InitOatDexFiles(oat_data_offset); + oat_size_ = InitOatHeader(instruction_set, + instruction_set_features, + dchecked_integral_cast<uint32_t>(oat_dex_files_.size()), + key_value_store); ChecksumUpdatingOutputStream checksum_updating_rodata(oat_rodata, oat_header_.get()); if (kIsVdexEnabled) { - std::unique_ptr<BufferedOutputStream> vdex_out( - MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(vdex_file))); + std::unique_ptr<BufferedOutputStream> vdex_out = + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file)); // Write DEX files into VDEX, mmap and open them. if (!WriteDexFiles(vdex_out.get(), vdex_file, update_input_vdex) || !OpenDexFiles(vdex_file, verify, &dex_files_map, &dex_files)) { @@ -534,18 +581,13 @@ bool OatWriter::WriteAndOpenDexFiles( } } - // Write TypeLookupTables into OAT. + // Write type lookup tables into the oat file. if (!WriteTypeLookupTables(&checksum_updating_rodata, dex_files)) { return false; } - // Reserve space for class offsets in OAT and update class_offsets_offset_. - for (OatDexFile& oat_dex_file : oat_dex_files_) { - oat_dex_file.ReserveClassOffsets(this); - } - - // Write OatDexFiles into OAT. Needs to be done last, once offsets are collected. - if (!WriteOatDexFiles(&checksum_updating_rodata)) { + // Write dex layout sections into the oat file. + if (!WriteDexLayoutSections(&checksum_updating_rodata, dex_files)) { return false; } @@ -567,16 +609,34 @@ void OatWriter::PrepareLayout(linker::MultiOatRelativePatcher* relative_patcher) InstructionSet instruction_set = compiler_driver_->GetInstructionSet(); CHECK_EQ(instruction_set, oat_header_->GetInstructionSet()); + { + TimingLogger::ScopedTiming split("InitBssLayout", timings_); + InitBssLayout(instruction_set); + } + uint32_t offset = oat_size_; { + TimingLogger::ScopedTiming split("InitClassOffsets", timings_); + offset = InitClassOffsets(offset); + } + { TimingLogger::ScopedTiming split("InitOatClasses", timings_); offset = InitOatClasses(offset); } { + TimingLogger::ScopedTiming split("InitMethodBssMappings", timings_); + offset = InitMethodBssMappings(offset); + } + { TimingLogger::ScopedTiming split("InitOatMaps", timings_); offset = InitOatMaps(offset); } { + TimingLogger::ScopedTiming split("InitOatDexFiles", timings_); + oat_header_->SetOatDexFilesOffset(offset); + offset = InitOatDexFiles(offset); + } + { TimingLogger::ScopedTiming split("InitOatCode", timings_); offset = InitOatCode(offset); } @@ -585,11 +645,7 @@ void OatWriter::PrepareLayout(linker::MultiOatRelativePatcher* relative_patcher) offset = InitOatCodeDexFiles(offset); } oat_size_ = offset; - - { - TimingLogger::ScopedTiming split("InitBssLayout", timings_); - InitBssLayout(instruction_set); - } + bss_start_ = (bss_size_ != 0u) ? RoundUp(oat_size_, kPageSize) : 0u; CHECK_EQ(dex_files_->size(), oat_dex_files_.size()); if (compiling_boot_image_) { @@ -606,11 +662,10 @@ OatWriter::~OatWriter() { class OatWriter::DexMethodVisitor { public: DexMethodVisitor(OatWriter* writer, size_t offset) - : writer_(writer), - offset_(offset), - dex_file_(nullptr), - class_def_index_(DexFile::kDexNoIndex) { - } + : writer_(writer), + offset_(offset), + dex_file_(nullptr), + class_def_index_(DexFile::kDexNoIndex) {} virtual bool StartClass(const DexFile* dex_file, size_t class_def_index) { DCHECK(dex_file_ == nullptr); @@ -650,19 +705,21 @@ class OatWriter::DexMethodVisitor { class OatWriter::OatDexMethodVisitor : public DexMethodVisitor { public: OatDexMethodVisitor(OatWriter* writer, size_t offset) - : DexMethodVisitor(writer, offset), - oat_class_index_(0u), - method_offsets_index_(0u) { - } + : DexMethodVisitor(writer, offset), + oat_class_index_(0u), + method_offsets_index_(0u) {} - bool StartClass(const DexFile* dex_file, size_t class_def_index) { + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE { DexMethodVisitor::StartClass(dex_file, class_def_index); - DCHECK_LT(oat_class_index_, writer_->oat_classes_.size()); + if (kIsDebugBuild && writer_->MayHaveCompiledMethods()) { + // There are no oat classes if there aren't any compiled methods. + CHECK_LT(oat_class_index_, writer_->oat_classes_.size()); + } method_offsets_index_ = 0u; return true; } - bool EndClass() { + bool EndClass() OVERRIDE { ++oat_class_index_; return DexMethodVisitor::EndClass(); } @@ -672,49 +729,107 @@ class OatWriter::OatDexMethodVisitor : public DexMethodVisitor { size_t method_offsets_index_; }; +static bool HasCompiledCode(const CompiledMethod* method) { + // The dextodexcompiler puts the quickening info table into the CompiledMethod + // for simplicity. For such methods, we will emit an OatQuickMethodHeader + // only when vdex is disabled. + return method != nullptr && (!method->GetQuickCode().empty() || !kIsVdexEnabled); +} + +static bool HasQuickeningInfo(const CompiledMethod* method) { + return method != nullptr && method->GetQuickCode().empty() && !method->GetVmapTable().empty(); +} + +class OatWriter::InitBssLayoutMethodVisitor : public DexMethodVisitor { + public: + explicit InitBssLayoutMethodVisitor(OatWriter* writer) + : DexMethodVisitor(writer, /* offset */ 0u) {} + + bool VisitMethod(size_t class_def_method_index ATTRIBUTE_UNUSED, + const ClassDataItemIterator& it) OVERRIDE { + // Look for patches with .bss references and prepare maps with placeholders for their offsets. + CompiledMethod* compiled_method = writer_->compiler_driver_->GetCompiledMethod( + MethodReference(dex_file_, it.GetMemberIndex())); + if (HasCompiledCode(compiled_method)) { + for (const LinkerPatch& patch : compiled_method->GetPatches()) { + if (patch.GetType() == LinkerPatch::Type::kMethodBssEntry) { + MethodReference target_method = patch.TargetMethod(); + auto refs_it = writer_->bss_method_entry_references_.find(target_method.dex_file); + if (refs_it == writer_->bss_method_entry_references_.end()) { + refs_it = writer_->bss_method_entry_references_.Put( + target_method.dex_file, + BitVector(target_method.dex_file->NumMethodIds(), + /* expandable */ false, + Allocator::GetMallocAllocator())); + refs_it->second.ClearAllBits(); + } + refs_it->second.SetBit(target_method.dex_method_index); + writer_->bss_method_entries_.Overwrite(target_method, /* placeholder */ 0u); + } else if (patch.GetType() == LinkerPatch::Type::kTypeBssEntry) { + TypeReference ref(patch.TargetTypeDexFile(), patch.TargetTypeIndex()); + writer_->bss_type_entries_.Overwrite(ref, /* placeholder */ 0u); + } else if (patch.GetType() == LinkerPatch::Type::kStringBssEntry) { + StringReference ref(patch.TargetStringDexFile(), patch.TargetStringIndex()); + writer_->bss_string_entries_.Overwrite(ref, /* placeholder */ 0u); + } + } + } else { + DCHECK(compiled_method == nullptr || compiled_method->GetPatches().empty()); + } + return true; + } +}; + class OatWriter::InitOatClassesMethodVisitor : public DexMethodVisitor { public: InitOatClassesMethodVisitor(OatWriter* writer, size_t offset) - : DexMethodVisitor(writer, offset), - compiled_methods_(), - num_non_null_compiled_methods_(0u) { + : DexMethodVisitor(writer, offset), + compiled_methods_(), + compiled_methods_with_code_(0u) { size_t num_classes = 0u; for (const OatDexFile& oat_dex_file : writer_->oat_dex_files_) { num_classes += oat_dex_file.class_offsets_.size(); } - writer_->oat_classes_.reserve(num_classes); + // If we aren't compiling only reserve headers. + writer_->oat_class_headers_.reserve(num_classes); + if (writer->MayHaveCompiledMethods()) { + writer->oat_classes_.reserve(num_classes); + } compiled_methods_.reserve(256u); + // If there are any classes, the class offsets allocation aligns the offset. + DCHECK(num_classes == 0u || IsAligned<4u>(offset)); } - bool StartClass(const DexFile* dex_file, size_t class_def_index) { + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE { DexMethodVisitor::StartClass(dex_file, class_def_index); compiled_methods_.clear(); - num_non_null_compiled_methods_ = 0u; + compiled_methods_with_code_ = 0u; return true; } bool VisitMethod(size_t class_def_method_index ATTRIBUTE_UNUSED, - const ClassDataItemIterator& it) { + const ClassDataItemIterator& it) OVERRIDE { // Fill in the compiled_methods_ array for methods that have a // CompiledMethod. We track the number of non-null entries in - // num_non_null_compiled_methods_ since we only want to allocate + // compiled_methods_with_code_ since we only want to allocate // OatMethodOffsets for the compiled methods. uint32_t method_idx = it.GetMemberIndex(); CompiledMethod* compiled_method = writer_->compiler_driver_->GetCompiledMethod(MethodReference(dex_file_, method_idx)); compiled_methods_.push_back(compiled_method); - if (compiled_method != nullptr) { - ++num_non_null_compiled_methods_; + if (HasCompiledCode(compiled_method)) { + ++compiled_methods_with_code_; } return true; } - bool EndClass() { + bool EndClass() OVERRIDE { ClassReference class_ref(dex_file_, class_def_index_); mirror::Class::Status status; bool found = writer_->compiler_driver_->GetCompiledClass(class_ref, &status); if (!found) { - if (writer_->compiler_driver_->GetVerificationResults()->IsClassRejected(class_ref)) { + VerificationResults* results = writer_->compiler_driver_->GetVerificationResults(); + if (results != nullptr && results->IsClassRejected(class_ref)) { // The oat class status is used only for verification of resolved classes, // so use kStatusErrorResolved whether the class was resolved or unresolved // during compile-time verification. @@ -724,30 +839,36 @@ class OatWriter::InitOatClassesMethodVisitor : public DexMethodVisitor { } } - writer_->oat_classes_.emplace_back(offset_, - compiled_methods_, - num_non_null_compiled_methods_, - status); - offset_ += writer_->oat_classes_.back().SizeOf(); + writer_->oat_class_headers_.emplace_back(offset_, + compiled_methods_with_code_, + compiled_methods_.size(), + status); + OatClassHeader& header = writer_->oat_class_headers_.back(); + offset_ += header.SizeOf(); + if (writer_->MayHaveCompiledMethods()) { + writer_->oat_classes_.emplace_back(compiled_methods_, + compiled_methods_with_code_, + header.type_); + offset_ += writer_->oat_classes_.back().SizeOf(); + } return DexMethodVisitor::EndClass(); } private: dchecked_vector<CompiledMethod*> compiled_methods_; - size_t num_non_null_compiled_methods_; + size_t compiled_methods_with_code_; }; class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { public: - InitCodeMethodVisitor(OatWriter* writer, size_t offset, size_t quickening_info_offset) - : OatDexMethodVisitor(writer, offset), - debuggable_(writer->GetCompilerDriver()->GetCompilerOptions().GetDebuggable()), - current_quickening_info_offset_(quickening_info_offset) { + InitCodeMethodVisitor(OatWriter* writer, size_t offset) + : OatDexMethodVisitor(writer, offset), + debuggable_(writer->GetCompilerDriver()->GetCompilerOptions().GetDebuggable()) { writer_->absolute_patch_locations_.reserve( writer_->compiler_driver_->GetNonRelativeLinkerPatchCount()); } - bool EndClass() { + bool EndClass() OVERRIDE { OatDexMethodVisitor::EndClass(); if (oat_class_index_ == writer_->oat_classes_.size()) { offset_ = writer_->relative_patcher_->ReserveSpaceEnd(offset_); @@ -755,15 +876,12 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { return true; } - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); - if (it.GetMethodCodeItem() != nullptr) { - current_quickening_info_offset_ += sizeof(uint32_t); - } - if (compiled_method != nullptr) { + if (HasCompiledCode(compiled_method)) { // Derived from CompiledMethod. uint32_t quick_code_offset = 0; @@ -824,18 +942,10 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { DCHECK_LT(method_info_offset, code_offset); } } else { - CHECK(compiled_method->GetMethodInfo().empty()); - if (kIsVdexEnabled) { - // We write the offset in the .vdex file. - DCHECK_EQ(vmap_table_offset, 0u); - vmap_table_offset = current_quickening_info_offset_; - ArrayRef<const uint8_t> vmap_table = compiled_method->GetVmapTable(); - current_quickening_info_offset_ += vmap_table.size() * sizeof(vmap_table.front()); - } else { - // We write the offset of the quickening info relative to the code. - vmap_table_offset += code_offset; - DCHECK_LT(vmap_table_offset, code_offset); - } + CHECK(!kIsVdexEnabled); + // We write the offset of the quickening info relative to the code. + vmap_table_offset += code_offset; + DCHECK_LT(vmap_table_offset, code_offset); } uint32_t frame_size_in_bytes = compiled_method->GetFrameSizeInBytes(); uint32_t core_spill_mask = compiled_method->GetCoreSpillMask(); @@ -858,14 +968,6 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { if (!patch.IsPcRelative()) { writer_->absolute_patch_locations_.push_back(base_loc + patch.LiteralOffset()); } - if (patch.GetType() == LinkerPatch::Type::kTypeBssEntry) { - TypeReference ref(patch.TargetTypeDexFile(), patch.TargetTypeIndex()); - writer_->bss_type_entries_.Overwrite(ref, /* placeholder */ 0u); - } - if (patch.GetType() == LinkerPatch::Type::kStringBssEntry) { - StringReference ref(patch.TargetStringDexFile(), patch.TargetStringIndex()); - writer_->bss_string_entries_.Overwrite(ref, /* placeholder */ 0u); - } } } } @@ -942,43 +1044,35 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { // Cache of compiler's --debuggable option. const bool debuggable_; - - // Offset in the vdex file for the quickening info. - uint32_t current_quickening_info_offset_; }; class OatWriter::InitMapMethodVisitor : public OatDexMethodVisitor { public: InitMapMethodVisitor(OatWriter* writer, size_t offset) - : OatDexMethodVisitor(writer, offset) { - } + : OatDexMethodVisitor(writer, offset) {} bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it ATTRIBUTE_UNUSED) - REQUIRES_SHARED(Locks::mutator_lock_) { + OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); - if (compiled_method != nullptr) { + if (HasCompiledCode(compiled_method)) { DCHECK_LT(method_offsets_index_, oat_class->method_offsets_.size()); - // If vdex is enabled, we only emit the stack map of compiled code. The quickening info will - // be in the vdex file. - if (!compiled_method->GetQuickCode().empty() || !kIsVdexEnabled) { - DCHECK_EQ(oat_class->method_headers_[method_offsets_index_].GetVmapTableOffset(), 0u); + DCHECK_EQ(oat_class->method_headers_[method_offsets_index_].GetVmapTableOffset(), 0u); - ArrayRef<const uint8_t> map = compiled_method->GetVmapTable(); - uint32_t map_size = map.size() * sizeof(map[0]); - if (map_size != 0u) { - size_t offset = dedupe_map_.GetOrCreate( - map.data(), - [this, map_size]() { - uint32_t new_offset = offset_; - offset_ += map_size; - return new_offset; - }); - // Code offset is not initialized yet, so set the map offset to 0u-offset. - DCHECK_EQ(oat_class->method_offsets_[method_offsets_index_].code_offset_, 0u); - oat_class->method_headers_[method_offsets_index_].SetVmapTableOffset(0u - offset); - } + ArrayRef<const uint8_t> map = compiled_method->GetVmapTable(); + uint32_t map_size = map.size() * sizeof(map[0]); + if (map_size != 0u) { + size_t offset = dedupe_map_.GetOrCreate( + map.data(), + [this, map_size]() { + uint32_t new_offset = offset_; + offset_ += map_size; + return new_offset; + }); + // Code offset is not initialized yet, so set the map offset to 0u-offset. + DCHECK_EQ(oat_class->method_offsets_[method_offsets_index_].code_offset_, 0u); + oat_class->method_headers_[method_offsets_index_].SetVmapTableOffset(0u - offset); } ++method_offsets_index_; } @@ -997,11 +1091,11 @@ class OatWriter::InitMethodInfoVisitor : public OatDexMethodVisitor { InitMethodInfoVisitor(OatWriter* writer, size_t offset) : OatDexMethodVisitor(writer, offset) {} bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it ATTRIBUTE_UNUSED) - REQUIRES_SHARED(Locks::mutator_lock_) { + OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); - if (compiled_method != nullptr) { + if (HasCompiledCode(compiled_method)) { DCHECK_LT(method_offsets_index_, oat_class->method_offsets_.size()); DCHECK_EQ(oat_class->method_headers_[method_offsets_index_].GetMethodInfoOffset(), 0u); ArrayRef<const uint8_t> map = compiled_method->GetMethodInfo(); @@ -1035,28 +1129,25 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { InitImageMethodVisitor(OatWriter* writer, size_t offset, const std::vector<const DexFile*>* dex_files) - : OatDexMethodVisitor(writer, offset), - pointer_size_(GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet())), - dex_files_(dex_files), - class_linker_(Runtime::Current()->GetClassLinker()) { - } + : OatDexMethodVisitor(writer, offset), + pointer_size_(GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet())), + class_loader_(writer->HasImage() ? writer->image_writer_->GetClassLoader() : nullptr), + dex_files_(dex_files), + class_linker_(Runtime::Current()->GetClassLinker()) {} // Handle copied methods here. Copy pointer to quick code from // an origin method to a copied method only if they are // in the same oat file. If the origin and the copied methods are // in different oat files don't touch the copied method. // References to other oat files are not supported yet. - bool StartClass(const DexFile* dex_file, size_t class_def_index) + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatDexMethodVisitor::StartClass(dex_file, class_def_index); // Skip classes that are not in the image. if (!IsImageClass()) { return true; } - ScopedObjectAccessUnchecked soa(Thread::Current()); - StackHandleScope<1> hs(soa.Self()); - Handle<mirror::DexCache> dex_cache = hs.NewHandle( - class_linker_->FindDexCache(Thread::Current(), *dex_file)); + ObjPtr<mirror::DexCache> dex_cache = class_linker_->FindDexCache(Thread::Current(), *dex_file); const DexFile::ClassDef& class_def = dex_file->GetClassDef(class_def_index); mirror::Class* klass = dex_cache->GetResolvedType(class_def.class_idx_); if (klass != nullptr) { @@ -1065,11 +1156,13 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { // in the copied method should be the same as in the origin // method. mirror::Class* declaring_class = method.GetDeclaringClass(); - ArtMethod* origin = declaring_class->FindDeclaredVirtualMethod( + ArtMethod* origin = declaring_class->FindClassMethod( declaring_class->GetDexCache(), method.GetDexMethodIndex(), pointer_size_); CHECK(origin != nullptr); + CHECK(!origin->IsDirect()); + CHECK(origin->GetDeclaringClass() == declaring_class); if (IsInOatFile(&declaring_class->GetDexFile())) { const void* code_ptr = origin->GetEntryPointFromQuickCompiledCodePtrSize(pointer_size_); @@ -1085,7 +1178,7 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { return true; } - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { // Skip methods that are not in the image. if (!IsImageClass()) { @@ -1096,43 +1189,42 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); OatMethodOffsets offsets(0u); - if (compiled_method != nullptr) { + if (HasCompiledCode(compiled_method)) { DCHECK_LT(method_offsets_index_, oat_class->method_offsets_.size()); offsets = oat_class->method_offsets_[method_offsets_index_]; ++method_offsets_index_; } - // Unchecked as we hold mutator_lock_ on entry. - ScopedObjectAccessUnchecked soa(Thread::Current()); - StackHandleScope<1> hs(soa.Self()); - Handle<mirror::DexCache> dex_cache(hs.NewHandle(class_linker_->FindDexCache( - Thread::Current(), *dex_file_))); + Thread* self = Thread::Current(); + ObjPtr<mirror::DexCache> dex_cache = class_linker_->FindDexCache(self, *dex_file_); ArtMethod* method; if (writer_->HasBootImage()) { const InvokeType invoke_type = it.GetMethodInvokeType( dex_file_->GetClassDef(class_def_index_)); - method = class_linker_->ResolveMethod<ClassLinker::kNoICCECheckForCache>( + // Unchecked as we hold mutator_lock_ on entry. + ScopedObjectAccessUnchecked soa(self); + StackHandleScope<1> hs(self); + method = class_linker_->ResolveMethod<ClassLinker::ResolveMode::kNoChecks>( *dex_file_, it.GetMemberIndex(), - dex_cache, + hs.NewHandle(dex_cache), ScopedNullHandle<mirror::ClassLoader>(), nullptr, invoke_type); if (method == nullptr) { LOG(FATAL_WITHOUT_ABORT) << "Unexpected failure to resolve a method: " << dex_file_->PrettyMethod(it.GetMemberIndex(), true); - soa.Self()->AssertPendingException(); - mirror::Throwable* exc = soa.Self()->GetException(); + self->AssertPendingException(); + mirror::Throwable* exc = self->GetException(); std::string dump = exc->Dump(); LOG(FATAL) << dump; UNREACHABLE(); } } else { - // Should already have been resolved by the compiler, just peek into the dex cache. + // Should already have been resolved by the compiler. // It may not be resolved if the class failed to verify, in this case, don't set the - // entrypoint. This is not fatal since the dex cache will contain a resolution method. - method = dex_cache->GetResolvedMethod(it.GetMemberIndex(), - class_linker_->GetImagePointerSize()); + // entrypoint. This is not fatal since we shall use a resolution method. + method = class_linker_->LookupResolvedMethod(it.GetMemberIndex(), dex_cache, class_loader_); } if (method != nullptr && compiled_method != nullptr && @@ -1171,8 +1263,9 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { } } - protected: + private: const PointerSize pointer_size_; + ObjPtr<mirror::ClassLoader> class_loader_; const std::vector<const DexFile*>* dex_files_; ClassLinker* const class_linker_; std::vector<std::pair<ArtMethod*, ArtMethod*>> methods_to_process_; @@ -1182,14 +1275,15 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { public: WriteCodeMethodVisitor(OatWriter* writer, OutputStream* out, const size_t file_offset, size_t relative_offset) SHARED_LOCK_FUNCTION(Locks::mutator_lock_) - : OatDexMethodVisitor(writer, relative_offset), - class_loader_(writer->HasImage() ? writer->image_writer_->GetClassLoader() : nullptr), - out_(out), - file_offset_(file_offset), - soa_(Thread::Current()), - no_thread_suspension_("OatWriter patching"), - class_linker_(Runtime::Current()->GetClassLinker()), - dex_cache_(nullptr) { + : OatDexMethodVisitor(writer, relative_offset), + pointer_size_(GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet())), + class_loader_(writer->HasImage() ? writer->image_writer_->GetClassLoader() : nullptr), + out_(out), + file_offset_(file_offset), + soa_(Thread::Current()), + no_thread_suspension_("OatWriter patching"), + class_linker_(Runtime::Current()->GetClassLinker()), + dex_cache_(nullptr) { patched_code_.reserve(16 * KB); if (writer_->HasBootImage()) { // If we're creating the image, the address space must be ready so that we can apply patches. @@ -1200,17 +1294,20 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { ~WriteCodeMethodVisitor() UNLOCK_FUNCTION(Locks::mutator_lock_) { } - bool StartClass(const DexFile* dex_file, size_t class_def_index) + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatDexMethodVisitor::StartClass(dex_file, class_def_index); - if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) { - dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file); - DCHECK(dex_cache_ != nullptr); + if (writer_->GetCompilerDriver()->GetCompilerOptions().IsAotCompilationEnabled()) { + // Only need to set the dex cache if we have compilation. Other modes might have unloaded it. + if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) { + dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file); + DCHECK(dex_cache_ != nullptr); + } } return true; } - bool EndClass() REQUIRES_SHARED(Locks::mutator_lock_) { + bool EndClass() OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { bool result = OatDexMethodVisitor::EndClass(); if (oat_class_index_ == writer_->oat_classes_.size()) { DCHECK(result); // OatDexMethodVisitor::EndClass() never fails. @@ -1223,14 +1320,14 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { return result; } - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; const CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); // No thread suspension since dex_cache_ that may get invalidated if that occurs. ScopedAssertNoThreadSuspension tsc(__FUNCTION__); - if (compiled_method != nullptr) { // ie. not an abstract method + if (HasCompiledCode(compiled_method)) { size_t file_offset = file_offset_; OutputStream* out = out_; @@ -1275,6 +1372,15 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { for (const LinkerPatch& patch : compiled_method->GetPatches()) { uint32_t literal_offset = patch.LiteralOffset(); switch (patch.GetType()) { + case LinkerPatch::Type::kMethodBssEntry: { + uint32_t target_offset = + writer_->bss_start_ + writer_->bss_method_entries_.Get(patch.TargetMethod()); + writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, + patch, + offset_ + literal_offset, + target_offset); + break; + } case LinkerPatch::Type::kCallRelative: { // NOTE: Relative calls across oat files are not supported. uint32_t target_offset = GetTargetOffset(patch); @@ -1284,14 +1390,6 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { target_offset); break; } - case LinkerPatch::Type::kDexCacheArray: { - uint32_t target_offset = GetDexCacheOffset(patch); - writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, - patch, - offset_ + literal_offset, - target_offset); - break; - } case LinkerPatch::Type::kStringRelative: { uint32_t target_offset = GetTargetObjectOffset(GetTargetString(patch)); writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, @@ -1302,7 +1400,8 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } case LinkerPatch::Type::kStringBssEntry: { StringReference ref(patch.TargetStringDexFile(), patch.TargetStringIndex()); - uint32_t target_offset = writer_->bss_string_entries_.Get(ref); + uint32_t target_offset = + writer_->bss_start_ + writer_->bss_string_entries_.Get(ref); writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, patch, offset_ + literal_offset, @@ -1319,7 +1418,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } case LinkerPatch::Type::kTypeBssEntry: { TypeReference ref(patch.TargetTypeDexFile(), patch.TargetTypeIndex()); - uint32_t target_offset = writer_->bss_type_entries_.Get(ref); + uint32_t target_offset = writer_->bss_start_ + writer_->bss_type_entries_.Get(ref); writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, patch, offset_ + literal_offset, @@ -1368,6 +1467,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } private: + const PointerSize pointer_size_; ObjPtr<mirror::ClassLoader> class_loader_; OutputStream* const out_; const size_t file_offset_; @@ -1388,8 +1488,8 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { ObjPtr<mirror::DexCache> dex_cache = (dex_file_ == ref.dex_file) ? dex_cache_ : class_linker_->FindDexCache( Thread::Current(), *ref.dex_file); - ArtMethod* method = dex_cache->GetResolvedMethod( - ref.dex_method_index, class_linker_->GetImagePointerSize()); + ArtMethod* method = + class_linker_->LookupResolvedMethod(ref.dex_method_index, dex_cache, class_loader_); CHECK(method != nullptr); return method; } @@ -1401,9 +1501,8 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { if (UNLIKELY(target_offset == 0)) { ArtMethod* target = GetTargetMethod(patch); DCHECK(target != nullptr); - PointerSize size = - GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet()); - const void* oat_code_offset = target->GetEntryPointFromQuickCompiledCodePtrSize(size); + const void* oat_code_offset = + target->GetEntryPointFromQuickCompiledCodePtrSize(pointer_size_); if (oat_code_offset != 0) { DCHECK(!writer_->HasBootImage()); DCHECK(!Runtime::Current()->GetClassLinker()->IsQuickResolutionStub(oat_code_offset)); @@ -1447,19 +1546,6 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { return string; } - uint32_t GetDexCacheOffset(const LinkerPatch& patch) REQUIRES_SHARED(Locks::mutator_lock_) { - if (writer_->HasBootImage()) { - uintptr_t element = writer_->image_writer_->GetDexCacheArrayElementImageAddress<uintptr_t>( - patch.TargetDexCacheDexFile(), patch.TargetDexCacheElementOffset()); - size_t oat_index = writer_->image_writer_->GetOatIndexForDexCache(dex_cache_); - uintptr_t oat_data = writer_->image_writer_->GetOatDataBegin(oat_index); - return element - oat_data; - } else { - size_t start = writer_->dex_cache_arrays_offsets_.Get(patch.TargetDexCacheDexFile()); - return start + patch.TargetDexCacheElementOffset(); - } - } - uint32_t GetTargetMethodOffset(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK(writer_->HasBootImage()); method = writer_->image_writer_->GetImageMethodAddress(method); @@ -1525,16 +1611,15 @@ class OatWriter::WriteMapMethodVisitor : public OatDexMethodVisitor { OutputStream* out, const size_t file_offset, size_t relative_offset) - : OatDexMethodVisitor(writer, relative_offset), - out_(out), - file_offset_(file_offset) { - } + : OatDexMethodVisitor(writer, relative_offset), + out_(out), + file_offset_(file_offset) {} - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) { + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; const CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); - if (compiled_method != nullptr) { // i.e. not an abstract method + if (HasCompiledCode(compiled_method)) { size_t file_offset = file_offset_; OutputStream* out = out_; @@ -1549,8 +1634,7 @@ class OatWriter::WriteMapMethodVisitor : public OatDexMethodVisitor { // If vdex is enabled, only emit the map for compiled code. The quickening info // is emitted in the vdex already. - if (map_offset != 0u && - !(kIsVdexEnabled && compiled_method->GetQuickCode().empty())) { + if (map_offset != 0u) { // Transform map_offset to actual oat data offset. map_offset = (code_offset - compiled_method->CodeDelta()) - map_offset; DCHECK_NE(map_offset, 0u); @@ -1589,15 +1673,15 @@ class OatWriter::WriteMethodInfoVisitor : public OatDexMethodVisitor { OutputStream* out, const size_t file_offset, size_t relative_offset) - : OatDexMethodVisitor(writer, relative_offset), - out_(out), - file_offset_(file_offset) {} + : OatDexMethodVisitor(writer, relative_offset), + out_(out), + file_offset_(file_offset) {} - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) { + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; const CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); - if (compiled_method != nullptr) { // i.e. not an abstract method + if (HasCompiledCode(compiled_method)) { size_t file_offset = file_offset_; OutputStream* out = out_; uint32_t map_offset = oat_class->method_headers_[method_offsets_index_].GetMethodInfoOffset(); @@ -1648,17 +1732,12 @@ bool OatWriter::VisitDexMethods(DexMethodVisitor* visitor) { if (UNLIKELY(!visitor->StartClass(dex_file, class_def_index))) { return false; } - if (compiler_driver_->GetCompilerOptions().IsAnyCompilationEnabled()) { + if (MayHaveCompiledMethods()) { const DexFile::ClassDef& class_def = dex_file->GetClassDef(class_def_index); const uint8_t* class_data = dex_file->GetClassData(class_def); if (class_data != nullptr) { // ie not an empty class, such as a marker interface ClassDataItemIterator it(*dex_file, class_data); - while (it.HasNextStaticField()) { - it.Next(); - } - while (it.HasNextInstanceField()) { - it.Next(); - } + it.SkipAllFields(); size_t class_def_method_index = 0u; while (it.HasNextDirectMethod()) { if (!visitor->VisitMethod(class_def_method_index, it)) { @@ -1698,12 +1777,17 @@ size_t OatWriter::InitOatHeader(InstructionSet instruction_set, return oat_header_->GetHeaderSize(); } -size_t OatWriter::InitOatDexFiles(size_t offset) { - TimingLogger::ScopedTiming split("InitOatDexFiles", timings_); - // Initialize offsets of dex files. +size_t OatWriter::InitClassOffsets(size_t offset) { + // Reserve space for class offsets in OAT and update class_offsets_offset_. for (OatDexFile& oat_dex_file : oat_dex_files_) { - oat_dex_file.offset_ = offset; - offset += oat_dex_file.SizeOf(); + DCHECK_EQ(oat_dex_file.class_offsets_offset_, 0u); + if (!oat_dex_file.class_offsets_.empty()) { + // Class offsets are required to be 4 byte aligned. + offset = RoundUp(offset, 4u); + oat_dex_file.class_offsets_offset_ = offset; + offset += oat_dex_file.GetClassOffsetsRawSize(); + DCHECK_ALIGNED(offset, 4u); + } } return offset; } @@ -1716,21 +1800,21 @@ size_t OatWriter::InitOatClasses(size_t offset) { offset = visitor.GetOffset(); // Update oat_dex_files_. - auto oat_class_it = oat_classes_.begin(); + auto oat_class_it = oat_class_headers_.begin(); for (OatDexFile& oat_dex_file : oat_dex_files_) { for (uint32_t& class_offset : oat_dex_file.class_offsets_) { - DCHECK(oat_class_it != oat_classes_.end()); + DCHECK(oat_class_it != oat_class_headers_.end()); class_offset = oat_class_it->offset_; ++oat_class_it; } } - CHECK(oat_class_it == oat_classes_.end()); + CHECK(oat_class_it == oat_class_headers_.end()); return offset; } size_t OatWriter::InitOatMaps(size_t offset) { - if (!compiler_driver_->GetCompilerOptions().IsAnyCompilationEnabled()) { + if (!MayHaveCompiledMethods()) { return offset; } { @@ -1748,6 +1832,50 @@ size_t OatWriter::InitOatMaps(size_t offset) { return offset; } +size_t OatWriter::InitMethodBssMappings(size_t offset) { + size_t number_of_dex_files = 0u; + for (size_t i = 0, size = dex_files_->size(); i != size; ++i) { + const DexFile* dex_file = (*dex_files_)[i]; + auto it = bss_method_entry_references_.find(dex_file); + if (it != bss_method_entry_references_.end()) { + const BitVector& method_indexes = it->second; + ++number_of_dex_files; + // If there are any classes, the class offsets allocation aligns the offset + // and we cannot have method bss mappings without class offsets. + static_assert(alignof(MethodBssMapping) == 4u, "MethodBssMapping alignment check."); + DCHECK_ALIGNED(offset, 4u); + oat_dex_files_[i].method_bss_mapping_offset_ = offset; + + linker::MethodBssMappingEncoder encoder( + GetInstructionSetPointerSize(oat_header_->GetInstructionSet())); + size_t number_of_entries = 0u; + bool first_index = true; + for (uint32_t method_index : method_indexes.Indexes()) { + uint32_t bss_offset = bss_method_entries_.Get(MethodReference(dex_file, method_index)); + if (first_index || !encoder.TryMerge(method_index, bss_offset)) { + encoder.Reset(method_index, bss_offset); + ++number_of_entries; + first_index = false; + } + } + DCHECK_NE(number_of_entries, 0u); + offset += MethodBssMapping::ComputeSize(number_of_entries); + } + } + // Check that all dex files targeted by method bss entries are in `*dex_files_`. + CHECK_EQ(number_of_dex_files, bss_method_entry_references_.size()); + return offset; +} + +size_t OatWriter::InitOatDexFiles(size_t offset) { + // Initialize offsets of oat dex files. + for (OatDexFile& oat_dex_file : oat_dex_files_) { + oat_dex_file.offset_ = offset; + offset += oat_dex_file.SizeOf(); + } + return offset; +} + size_t OatWriter::InitOatCode(size_t offset) { // calculate the offsets within OatHeader to executable code size_t old_offset = offset; @@ -1789,7 +1917,7 @@ size_t OatWriter::InitOatCodeDexFiles(size_t offset) { if (!compiler_driver_->GetCompilerOptions().IsAnyCompilationEnabled()) { return offset; } - InitCodeMethodVisitor code_visitor(this, offset, vdex_quickening_info_offset_); + InitCodeMethodVisitor code_visitor(this, offset); bool success = VisitDexMethods(&code_visitor); DCHECK(success); offset = code_visitor.GetOffset(); @@ -1806,38 +1934,51 @@ size_t OatWriter::InitOatCodeDexFiles(size_t offset) { } void OatWriter::InitBssLayout(InstructionSet instruction_set) { + { + InitBssLayoutMethodVisitor visitor(this); + bool success = VisitDexMethods(&visitor); + DCHECK(success); + } + + DCHECK_EQ(bss_size_, 0u); if (HasBootImage()) { DCHECK(bss_string_entries_.empty()); - if (bss_type_entries_.empty()) { + if (bss_method_entries_.empty() && bss_type_entries_.empty()) { // Nothing to put to the .bss section. return; } } // Allocate space for app dex cache arrays in the .bss section. - bss_start_ = RoundUp(oat_size_, kPageSize); - bss_size_ = 0u; + PointerSize pointer_size = GetInstructionSetPointerSize(instruction_set); if (!HasBootImage()) { - PointerSize pointer_size = GetInstructionSetPointerSize(instruction_set); for (const DexFile* dex_file : *dex_files_) { - dex_cache_arrays_offsets_.Put(dex_file, bss_start_ + bss_size_); DexCacheArraysLayout layout(pointer_size, dex_file); bss_size_ += layout.Size(); } } + bss_methods_offset_ = bss_size_; + + // Prepare offsets for .bss ArtMethod entries. + for (auto& entry : bss_method_entries_) { + DCHECK_EQ(entry.second, 0u); + entry.second = bss_size_; + bss_size_ += static_cast<size_t>(pointer_size); + } + bss_roots_offset_ = bss_size_; // Prepare offsets for .bss Class entries. for (auto& entry : bss_type_entries_) { DCHECK_EQ(entry.second, 0u); - entry.second = bss_start_ + bss_size_; + entry.second = bss_size_; bss_size_ += sizeof(GcRoot<mirror::Class>); } // Prepare offsets for .bss String entries. for (auto& entry : bss_string_entries_) { DCHECK_EQ(entry.second, 0u); - entry.second = bss_start_ + bss_size_; + entry.second = bss_size_; bss_size_ += sizeof(GcRoot<mirror::String>); } } @@ -1845,30 +1986,45 @@ void OatWriter::InitBssLayout(InstructionSet instruction_set) { bool OatWriter::WriteRodata(OutputStream* out) { CHECK(write_state_ == WriteState::kWriteRoData); + size_t file_offset = oat_data_offset_; + off_t current_offset = out->Seek(0, kSeekCurrent); + if (current_offset == static_cast<off_t>(-1)) { + PLOG(ERROR) << "Failed to retrieve current position in " << out->GetLocation(); + } + DCHECK_GE(static_cast<size_t>(current_offset), file_offset + oat_header_->GetHeaderSize()); + size_t relative_offset = current_offset - file_offset; + // Wrap out to update checksum with each write. ChecksumUpdatingOutputStream checksum_updating_out(out, oat_header_.get()); out = &checksum_updating_out; - if (!WriteClassOffsets(out)) { - LOG(ERROR) << "Failed to write class offsets to " << out->GetLocation(); + relative_offset = WriteClassOffsets(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write class offsets to " << out->GetLocation(); return false; } - if (!WriteClasses(out)) { - LOG(ERROR) << "Failed to write classes to " << out->GetLocation(); + relative_offset = WriteClasses(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write classes to " << out->GetLocation(); return false; } - off_t tables_end_offset = out->Seek(0, kSeekCurrent); - if (tables_end_offset == static_cast<off_t>(-1)) { - LOG(ERROR) << "Failed to get oat code position in " << out->GetLocation(); + relative_offset = WriteMethodBssMappings(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write method bss mappings to " << out->GetLocation(); return false; } - size_t file_offset = oat_data_offset_; - size_t relative_offset = static_cast<size_t>(tables_end_offset) - file_offset; + relative_offset = WriteMaps(out, file_offset, relative_offset); if (relative_offset == 0) { - LOG(ERROR) << "Failed to write oat code to " << out->GetLocation(); + PLOG(ERROR) << "Failed to write oat code to " << out->GetLocation(); + return false; + } + + relative_offset = WriteOatDexFiles(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write oat dex information to " << out->GetLocation(); return false; } @@ -1890,39 +2046,102 @@ bool OatWriter::WriteRodata(OutputStream* out) { class OatWriter::WriteQuickeningInfoMethodVisitor : public DexMethodVisitor { public: - WriteQuickeningInfoMethodVisitor(OatWriter* writer, OutputStream* out, uint32_t offset) - : DexMethodVisitor(writer, offset), - out_(out), - written_bytes_(0u) {} - - bool VisitMethod(size_t class_def_method_index ATTRIBUTE_UNUSED, - const ClassDataItemIterator& it) { - if (it.GetMethodCodeItem() == nullptr) { - // No CodeItem. Native or abstract method. - return true; - } - + WriteQuickeningInfoMethodVisitor(OatWriter* writer, + OutputStream* out, + uint32_t offset, + SafeMap<const uint8_t*, uint32_t>* offset_map) + : DexMethodVisitor(writer, offset), + out_(out), + written_bytes_(0u), + offset_map_(offset_map) {} + + bool VisitMethod(size_t class_def_method_index ATTRIBUTE_UNUSED, const ClassDataItemIterator& it) + OVERRIDE { uint32_t method_idx = it.GetMemberIndex(); CompiledMethod* compiled_method = writer_->compiler_driver_->GetCompiledMethod(MethodReference(dex_file_, method_idx)); - uint32_t length = 0; - const uint8_t* data = nullptr; - // VMap only contains quickening info if this method is not compiled. - if (compiled_method != nullptr && compiled_method->GetQuickCode().empty()) { + if (HasQuickeningInfo(compiled_method)) { ArrayRef<const uint8_t> map = compiled_method->GetVmapTable(); - data = map.data(); - length = map.size() * sizeof(map.front()); + // Deduplication is already done on a pointer basis by the compiler driver, + // so we can simply compare the pointers to find out if things are duplicated. + if (offset_map_->find(map.data()) == offset_map_->end()) { + uint32_t length = map.size() * sizeof(map.front()); + offset_map_->Put(map.data(), written_bytes_); + if (!out_->WriteFully(&length, sizeof(length)) || + !out_->WriteFully(map.data(), length)) { + PLOG(ERROR) << "Failed to write quickening info for " + << dex_file_->PrettyMethod(it.GetMemberIndex()) << " to " + << out_->GetLocation(); + return false; + } + written_bytes_ += sizeof(length) + length; + offset_ += sizeof(length) + length; + } } - if (!out_->WriteFully(&length, sizeof(length)) || - !out_->WriteFully(data, length)) { - PLOG(ERROR) << "Failed to write quickening info for " - << dex_file_->PrettyMethod(it.GetMemberIndex()) << " to " << out_->GetLocation(); - return false; + return true; + } + + size_t GetNumberOfWrittenBytes() const { + return written_bytes_; + } + + private: + OutputStream* const out_; + size_t written_bytes_; + // Maps quickening map to its offset in the file. + SafeMap<const uint8_t*, uint32_t>* offset_map_; +}; + +class OatWriter::WriteQuickeningIndicesMethodVisitor { + public: + WriteQuickeningIndicesMethodVisitor(OutputStream* out, + uint32_t indices_offset, + const SafeMap<const uint8_t*, uint32_t>& offset_map, + std::vector<uint32_t>* dex_files_offset) + : out_(out), + indices_offset_(indices_offset), + written_bytes_(0u), + dex_files_offset_(dex_files_offset), + offset_map_(offset_map) {} + + bool VisitDexMethods(const std::vector<const DexFile*>& dex_files, const CompilerDriver& driver) { + for (const DexFile* dex_file : dex_files) { + // Record the offset for this current dex file. It will be written in the vdex file + // later. + dex_files_offset_->push_back(indices_offset_ + GetNumberOfWrittenBytes()); + const size_t class_def_count = dex_file->NumClassDefs(); + for (size_t class_def_index = 0; class_def_index != class_def_count; ++class_def_index) { + const DexFile::ClassDef& class_def = dex_file->GetClassDef(class_def_index); + const uint8_t* class_data = dex_file->GetClassData(class_def); + if (class_data == nullptr) { + continue; + } + for (ClassDataItemIterator class_it(*dex_file, class_data); + class_it.HasNext(); + class_it.Next()) { + if (!class_it.IsAtMethod()) { + continue; + } + uint32_t method_idx = class_it.GetMemberIndex(); + CompiledMethod* compiled_method = + driver.GetCompiledMethod(MethodReference(dex_file, method_idx)); + if (HasQuickeningInfo(compiled_method)) { + uint32_t code_item_offset = class_it.GetMethodCodeItemOffset(); + uint32_t offset = offset_map_.Get(compiled_method->GetVmapTable().data()); + if (!out_->WriteFully(&code_item_offset, sizeof(code_item_offset)) || + !out_->WriteFully(&offset, sizeof(offset))) { + PLOG(ERROR) << "Failed to write quickening info for " + << dex_file->PrettyMethod(method_idx) << " to " + << out_->GetLocation(); + return false; + } + written_bytes_ += sizeof(code_item_offset) + sizeof(offset); + } + } + } } - offset_ += sizeof(length) + length; - written_bytes_ += sizeof(length) + length; return true; } @@ -1932,7 +2151,11 @@ class OatWriter::WriteQuickeningInfoMethodVisitor : public DexMethodVisitor { private: OutputStream* const out_; + const uint32_t indices_offset_; size_t written_bytes_; + std::vector<uint32_t>* dex_files_offset_; + // Maps quickening map to its offset in the file. + const SafeMap<const uint8_t*, uint32_t>& offset_map_; }; bool OatWriter::WriteQuickeningInfo(OutputStream* vdex_out) { @@ -1956,8 +2179,26 @@ bool OatWriter::WriteQuickeningInfo(OutputStream* vdex_out) { } if (compiler_driver_->GetCompilerOptions().IsAnyCompilationEnabled()) { - WriteQuickeningInfoMethodVisitor visitor(this, vdex_out, start_offset); - if (!VisitDexMethods(&visitor)) { + std::vector<uint32_t> dex_files_indices; + SafeMap<const uint8_t*, uint32_t> offset_map; + WriteQuickeningInfoMethodVisitor visitor1(this, vdex_out, start_offset, &offset_map); + if (!VisitDexMethods(&visitor1)) { + PLOG(ERROR) << "Failed to write the vdex quickening info. File: " << vdex_out->GetLocation(); + return false; + } + + WriteQuickeningIndicesMethodVisitor visitor2(vdex_out, + visitor1.GetNumberOfWrittenBytes(), + offset_map, + &dex_files_indices); + if (!visitor2.VisitDexMethods(*dex_files_, *compiler_driver_)) { + PLOG(ERROR) << "Failed to write the vdex quickening info. File: " << vdex_out->GetLocation(); + return false; + } + + DCHECK_EQ(dex_files_->size(), dex_files_indices.size()); + if (!vdex_out->WriteFully( + dex_files_indices.data(), sizeof(dex_files_indices[0]) * dex_files_indices.size())) { PLOG(ERROR) << "Failed to write the vdex quickening info. File: " << vdex_out->GetLocation(); return false; } @@ -1967,7 +2208,9 @@ bool OatWriter::WriteQuickeningInfo(OutputStream* vdex_out) { << " File: " << vdex_out->GetLocation(); return false; } - size_quickening_info_ = visitor.GetNumberOfWrittenBytes(); + size_quickening_info_ = visitor1.GetNumberOfWrittenBytes() + + visitor2.GetNumberOfWrittenBytes() + + dex_files_->size() * sizeof(uint32_t); } else { // We know we did not quicken. size_quickening_info_ = 0; @@ -2092,6 +2335,10 @@ bool OatWriter::WriteCode(OutputStream* out) { DO_STAT(size_oat_dex_file_offset_); DO_STAT(size_oat_dex_file_class_offsets_offset_); DO_STAT(size_oat_dex_file_lookup_table_offset_); + DO_STAT(size_oat_dex_file_dex_layout_sections_offset_); + DO_STAT(size_oat_dex_file_dex_layout_sections_); + DO_STAT(size_oat_dex_file_dex_layout_sections_alignment_); + DO_STAT(size_oat_dex_file_method_bss_mapping_offset_); DO_STAT(size_oat_lookup_table_alignment_); DO_STAT(size_oat_lookup_table_); DO_STAT(size_oat_class_offsets_alignment_); @@ -2100,6 +2347,7 @@ bool OatWriter::WriteCode(OutputStream* out) { DO_STAT(size_oat_class_status_); DO_STAT(size_oat_class_method_bitmaps_); DO_STAT(size_oat_class_method_offsets_); + DO_STAT(size_method_bss_mappings_); #undef DO_STAT VLOG(compiler) << "size_total=" << PrettySize(size_total) << " (" << size_total << "B)"; @@ -2172,35 +2420,51 @@ bool OatWriter::WriteHeader(OutputStream* out, return true; } -bool OatWriter::WriteClassOffsets(OutputStream* out) { +size_t OatWriter::WriteClassOffsets(OutputStream* out, size_t file_offset, size_t relative_offset) { for (OatDexFile& oat_dex_file : oat_dex_files_) { if (oat_dex_file.class_offsets_offset_ != 0u) { - uint32_t expected_offset = oat_data_offset_ + oat_dex_file.class_offsets_offset_; - off_t actual_offset = out->Seek(expected_offset, kSeekSet); - if (static_cast<uint32_t>(actual_offset) != expected_offset) { - PLOG(ERROR) << "Failed to seek to oat class offsets section. Actual: " << actual_offset - << " Expected: " << expected_offset << " File: " << oat_dex_file.GetLocation(); - return false; + // Class offsets are required to be 4 byte aligned. + if (UNLIKELY(!IsAligned<4u>(relative_offset))) { + size_t padding_size = RoundUp(relative_offset, 4u) - relative_offset; + if (!WriteUpTo16BytesAlignment(out, padding_size, &size_oat_class_offsets_alignment_)) { + return 0u; + } + relative_offset += padding_size; } + DCHECK_OFFSET(); if (!oat_dex_file.WriteClassOffsets(this, out)) { - return false; + return 0u; } + relative_offset += oat_dex_file.GetClassOffsetsRawSize(); } } - return true; + return relative_offset; } -bool OatWriter::WriteClasses(OutputStream* out) { - for (OatClass& oat_class : oat_classes_) { - if (!oat_class.Write(this, out, oat_data_offset_)) { - PLOG(ERROR) << "Failed to write oat methods information to " << out->GetLocation(); - return false; +size_t OatWriter::WriteClasses(OutputStream* out, size_t file_offset, size_t relative_offset) { + const bool may_have_compiled = MayHaveCompiledMethods(); + if (may_have_compiled) { + CHECK_EQ(oat_class_headers_.size(), oat_classes_.size()); + } + for (size_t i = 0; i < oat_class_headers_.size(); ++i) { + // If there are any classes, the class offsets allocation aligns the offset. + DCHECK_ALIGNED(relative_offset, 4u); + DCHECK_OFFSET(); + if (!oat_class_headers_[i].Write(this, out, oat_data_offset_)) { + return 0u; + } + relative_offset += oat_class_headers_[i].SizeOf(); + if (may_have_compiled) { + if (!oat_classes_[i].Write(this, out)) { + return 0u; + } + relative_offset += oat_classes_[i].SizeOf(); } } - return true; + return relative_offset; } -size_t OatWriter::WriteMaps(OutputStream* out, const size_t file_offset, size_t relative_offset) { +size_t OatWriter::WriteMaps(OutputStream* out, size_t file_offset, size_t relative_offset) { { size_t vmap_tables_offset = relative_offset; WriteMapMethodVisitor visitor(this, out, file_offset, relative_offset); @@ -2223,7 +2487,87 @@ size_t OatWriter::WriteMaps(OutputStream* out, const size_t file_offset, size_t return relative_offset; } -size_t OatWriter::WriteCode(OutputStream* out, const size_t file_offset, size_t relative_offset) { +size_t OatWriter::WriteMethodBssMappings(OutputStream* out, + size_t file_offset, + size_t relative_offset) { + TimingLogger::ScopedTiming split("WriteMethodBssMappings", timings_); + + for (size_t i = 0, size = dex_files_->size(); i != size; ++i) { + const DexFile* dex_file = (*dex_files_)[i]; + OatDexFile* oat_dex_file = &oat_dex_files_[i]; + auto it = bss_method_entry_references_.find(dex_file); + if (it != bss_method_entry_references_.end()) { + const BitVector& method_indexes = it->second; + // If there are any classes, the class offsets allocation aligns the offset + // and we cannot have method bss mappings without class offsets. + static_assert(alignof(MethodBssMapping) == sizeof(uint32_t), + "MethodBssMapping alignment check."); + DCHECK_ALIGNED(relative_offset, sizeof(uint32_t)); + + linker::MethodBssMappingEncoder encoder( + GetInstructionSetPointerSize(oat_header_->GetInstructionSet())); + // Allocate a sufficiently large MethodBssMapping. + size_t number_of_method_indexes = method_indexes.NumSetBits(); + DCHECK_NE(number_of_method_indexes, 0u); + size_t max_mappings_size = MethodBssMapping::ComputeSize(number_of_method_indexes); + DCHECK_ALIGNED(max_mappings_size, sizeof(uint32_t)); + std::unique_ptr<uint32_t[]> storage(new uint32_t[max_mappings_size / sizeof(uint32_t)]); + MethodBssMapping* mappings = new(storage.get()) MethodBssMapping(number_of_method_indexes); + mappings->ClearPadding(); + // Encode the MethodBssMapping. + auto init_it = mappings->begin(); + bool first_index = true; + for (uint32_t method_index : method_indexes.Indexes()) { + size_t bss_offset = bss_method_entries_.Get(MethodReference(dex_file, method_index)); + if (first_index) { + first_index = false; + encoder.Reset(method_index, bss_offset); + } else if (!encoder.TryMerge(method_index, bss_offset)) { + *init_it = encoder.GetEntry(); + ++init_it; + encoder.Reset(method_index, bss_offset); + } + } + // Store the last entry and shrink the mapping to the actual size. + *init_it = encoder.GetEntry(); + ++init_it; + DCHECK(init_it <= mappings->end()); + mappings->SetSize(std::distance(mappings->begin(), init_it)); + size_t mappings_size = MethodBssMapping::ComputeSize(mappings->size()); + + DCHECK_EQ(relative_offset, oat_dex_file->method_bss_mapping_offset_); + DCHECK_OFFSET(); + if (!out->WriteFully(storage.get(), mappings_size)) { + return 0u; + } + size_method_bss_mappings_ += mappings_size; + relative_offset += mappings_size; + } else { + DCHECK_EQ(0u, oat_dex_file->method_bss_mapping_offset_); + } + } + return relative_offset; +} + +size_t OatWriter::WriteOatDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset) { + TimingLogger::ScopedTiming split("WriteOatDexFiles", timings_); + + for (size_t i = 0, size = oat_dex_files_.size(); i != size; ++i) { + OatDexFile* oat_dex_file = &oat_dex_files_[i]; + DCHECK_EQ(relative_offset, oat_dex_file->offset_); + DCHECK_OFFSET(); + + // Write OatDexFile. + if (!oat_dex_file->Write(this, out)) { + return 0u; + } + relative_offset += oat_dex_file->SizeOf(); + } + + return relative_offset; +} + +size_t OatWriter::WriteCode(OutputStream* out, size_t file_offset, size_t relative_offset) { if (compiler_driver_->GetCompilerOptions().IsBootImage()) { InstructionSet instruction_set = compiler_driver_->GetInstructionSet(); @@ -2253,7 +2597,7 @@ size_t OatWriter::WriteCode(OutputStream* out, const size_t file_offset, size_t } size_t OatWriter::WriteCodeDexFiles(OutputStream* out, - const size_t file_offset, + size_t file_offset, size_t relative_offset) { #define VISIT(VisitorType) \ do { \ @@ -2356,7 +2700,7 @@ bool OatWriter::WriteDexFile(OutputStream* out, return false; } if (profile_compilation_info_ != nullptr) { - DCHECK(!update_input_vdex); + CHECK(!update_input_vdex) << "We should never update the input vdex when doing dexlayout"; if (!LayoutAndWriteDexFile(out, oat_dex_file)) { return false; } @@ -2482,6 +2826,7 @@ bool OatWriter::LayoutAndWriteDexFile(OutputStream* out, OatDexFile* oat_dex_fil if (!WriteDexFile(out, oat_dex_file, mem_map->Begin(), /* update_input_vdex */ false)) { return false; } + oat_dex_file->dex_sections_layout_ = dex_layout.GetSections(); // Set the checksum of the new oat dex file to be the original file's checksum. oat_dex_file->dex_file_location_checksum_ = dex_file->GetLocationChecksum(); return true; @@ -2667,50 +3012,6 @@ bool OatWriter::WriteDexFile(OutputStream* out, return true; } -bool OatWriter::WriteOatDexFiles(OutputStream* rodata) { - TimingLogger::ScopedTiming split("WriteOatDexFiles", timings_); - - off_t initial_offset = rodata->Seek(0, kSeekCurrent); - if (initial_offset == static_cast<off_t>(-1)) { - LOG(ERROR) << "Failed to get current position in " << rodata->GetLocation(); - return false; - } - - // Seek to the start of OatDexFiles, i.e. to the end of the OatHeader. If there are - // no OatDexFiles, no data is actually written to .rodata before WriteHeader() and - // this Seek() ensures that we reserve the space for OatHeader in .rodata. - DCHECK(oat_dex_files_.empty() || oat_dex_files_[0u].offset_ == oat_header_->GetHeaderSize()); - uint32_t expected_offset = oat_data_offset_ + oat_header_->GetHeaderSize(); - off_t actual_offset = rodata->Seek(expected_offset, kSeekSet); - if (static_cast<uint32_t>(actual_offset) != expected_offset) { - PLOG(ERROR) << "Failed to seek to OatDexFile table section. Actual: " << actual_offset - << " Expected: " << expected_offset << " File: " << rodata->GetLocation(); - return false; - } - - for (size_t i = 0, size = oat_dex_files_.size(); i != size; ++i) { - OatDexFile* oat_dex_file = &oat_dex_files_[i]; - - DCHECK_EQ(oat_data_offset_ + oat_dex_file->offset_, - static_cast<size_t>(rodata->Seek(0, kSeekCurrent))); - - // Write OatDexFile. - if (!oat_dex_file->Write(this, rodata)) { - PLOG(ERROR) << "Failed to write oat dex information to " << rodata->GetLocation(); - return false; - } - } - - // Seek back to the initial position. - if (rodata->Seek(initial_offset, kSeekSet) != initial_offset) { - PLOG(ERROR) << "Failed to seek to initial position. Actual: " << actual_offset - << " Expected: " << initial_offset << " File: " << rodata->GetLocation(); - return false; - } - - return true; -} - bool OatWriter::OpenDexFiles( File* file, bool verify, @@ -2871,6 +3172,70 @@ bool OatWriter::WriteTypeLookupTables( return true; } +bool OatWriter::WriteDexLayoutSections( + OutputStream* oat_rodata, + const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files) { + TimingLogger::ScopedTiming split(__FUNCTION__, timings_); + + if (!kWriteDexLayoutInfo) { + return true;; + } + + uint32_t expected_offset = oat_data_offset_ + oat_size_; + off_t actual_offset = oat_rodata->Seek(expected_offset, kSeekSet); + if (static_cast<uint32_t>(actual_offset) != expected_offset) { + PLOG(ERROR) << "Failed to seek to dex layout section offset section. Actual: " << actual_offset + << " Expected: " << expected_offset << " File: " << oat_rodata->GetLocation(); + return false; + } + + DCHECK_EQ(opened_dex_files.size(), oat_dex_files_.size()); + size_t rodata_offset = oat_size_; + for (size_t i = 0, size = opened_dex_files.size(); i != size; ++i) { + OatDexFile* oat_dex_file = &oat_dex_files_[i]; + DCHECK_EQ(oat_dex_file->dex_sections_layout_offset_, 0u); + + // Write dex layout section alignment bytes. + const size_t padding_size = + RoundUp(rodata_offset, alignof(DexLayoutSections)) - rodata_offset; + if (padding_size != 0u) { + std::vector<uint8_t> buffer(padding_size, 0u); + if (!oat_rodata->WriteFully(buffer.data(), padding_size)) { + PLOG(ERROR) << "Failed to write lookup table alignment padding." + << " File: " << oat_dex_file->GetLocation() + << " Output: " << oat_rodata->GetLocation(); + return false; + } + size_oat_dex_file_dex_layout_sections_alignment_ += padding_size; + rodata_offset += padding_size; + } + + DCHECK_ALIGNED(rodata_offset, alignof(DexLayoutSections)); + DCHECK_EQ(oat_data_offset_ + rodata_offset, + static_cast<size_t>(oat_rodata->Seek(0u, kSeekCurrent))); + DCHECK(oat_dex_file != nullptr); + if (!oat_rodata->WriteFully(&oat_dex_file->dex_sections_layout_, + sizeof(oat_dex_file->dex_sections_layout_))) { + PLOG(ERROR) << "Failed to write dex layout sections." + << " File: " << oat_dex_file->GetLocation() + << " Output: " << oat_rodata->GetLocation(); + return false; + } + oat_dex_file->dex_sections_layout_offset_ = rodata_offset; + size_oat_dex_file_dex_layout_sections_ += sizeof(oat_dex_file->dex_sections_layout_); + rodata_offset += sizeof(oat_dex_file->dex_sections_layout_); + } + oat_size_ = rodata_offset; + + if (!oat_rodata->Flush()) { + PLOG(ERROR) << "Failed to flush stream after writing type dex layout sections." + << " File: " << oat_rodata->GetLocation(); + return false; + } + + return true; +} + bool OatWriter::WriteChecksumsAndVdexHeader(OutputStream* vdex_out) { if (!kIsVdexEnabled) { return true; @@ -2929,14 +3294,18 @@ bool OatWriter::WriteChecksumsAndVdexHeader(OutputStream* vdex_out) { } bool OatWriter::WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta) { + return WriteUpTo16BytesAlignment(out, aligned_code_delta, &size_code_alignment_); +} + +bool OatWriter::WriteUpTo16BytesAlignment(OutputStream* out, uint32_t size, uint32_t* stat) { static const uint8_t kPadding[] = { 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u }; - DCHECK_LE(aligned_code_delta, sizeof(kPadding)); - if (UNLIKELY(!out->WriteFully(kPadding, aligned_code_delta))) { + DCHECK_LE(size, sizeof(kPadding)); + if (UNLIKELY(!out->WriteFully(kPadding, size))) { return false; } - size_code_alignment_ += aligned_code_delta; + *stat += size; return true; } @@ -2965,6 +3334,8 @@ OatWriter::OatDexFile::OatDexFile(const char* dex_file_location, dex_file_offset_(0u), class_offsets_offset_(0u), lookup_table_offset_(0u), + method_bss_mapping_offset_(0u), + dex_sections_layout_offset_(0u), class_offsets_() { } @@ -2974,19 +3345,9 @@ size_t OatWriter::OatDexFile::SizeOf() const { + sizeof(dex_file_location_checksum_) + sizeof(dex_file_offset_) + sizeof(class_offsets_offset_) - + sizeof(lookup_table_offset_); -} - -void OatWriter::OatDexFile::ReserveClassOffsets(OatWriter* oat_writer) { - DCHECK_EQ(class_offsets_offset_, 0u); - if (!class_offsets_.empty()) { - // Class offsets are required to be 4 byte aligned. - size_t initial_offset = oat_writer->oat_size_; - size_t offset = RoundUp(initial_offset, 4); - oat_writer->size_oat_class_offsets_alignment_ += offset - initial_offset; - class_offsets_offset_ = offset; - oat_writer->oat_size_ = offset + GetClassOffsetsRawSize(); - } + + sizeof(lookup_table_offset_) + + sizeof(method_bss_mapping_offset_) + + sizeof(dex_sections_layout_offset_); } bool OatWriter::OatDexFile::Write(OatWriter* oat_writer, OutputStream* out) const { @@ -3029,6 +3390,18 @@ bool OatWriter::OatDexFile::Write(OatWriter* oat_writer, OutputStream* out) cons } oat_writer->size_oat_dex_file_lookup_table_offset_ += sizeof(lookup_table_offset_); + if (!out->WriteFully(&dex_sections_layout_offset_, sizeof(dex_sections_layout_offset_))) { + PLOG(ERROR) << "Failed to write dex section layout info to " << out->GetLocation(); + return false; + } + oat_writer->size_oat_dex_file_dex_layout_sections_offset_ += sizeof(dex_sections_layout_offset_); + + if (!out->WriteFully(&method_bss_mapping_offset_, sizeof(method_bss_mapping_offset_))) { + PLOG(ERROR) << "Failed to write method bss mapping offset to " << out->GetLocation(); + return false; + } + oat_writer->size_oat_dex_file_method_bss_mapping_offset_ += sizeof(method_bss_mapping_offset_); + return true; } @@ -3042,37 +3415,21 @@ bool OatWriter::OatDexFile::WriteClassOffsets(OatWriter* oat_writer, OutputStrea return true; } -OatWriter::OatClass::OatClass(size_t offset, - const dchecked_vector<CompiledMethod*>& compiled_methods, - uint32_t num_non_null_compiled_methods, - mirror::Class::Status status) +OatWriter::OatClass::OatClass(const dchecked_vector<CompiledMethod*>& compiled_methods, + uint32_t compiled_methods_with_code, + uint16_t oat_class_type) : compiled_methods_(compiled_methods) { - uint32_t num_methods = compiled_methods.size(); - CHECK_LE(num_non_null_compiled_methods, num_methods); + const uint32_t num_methods = compiled_methods.size(); + CHECK_LE(compiled_methods_with_code, num_methods); - offset_ = offset; oat_method_offsets_offsets_from_oat_class_.resize(num_methods); - // Since both kOatClassNoneCompiled and kOatClassAllCompiled could - // apply when there are 0 methods, we just arbitrarily say that 0 - // methods means kOatClassNoneCompiled and that we won't use - // kOatClassAllCompiled unless there is at least one compiled - // method. This means in an interpretter only system, we can assert - // that all classes are kOatClassNoneCompiled. - if (num_non_null_compiled_methods == 0) { - type_ = kOatClassNoneCompiled; - } else if (num_non_null_compiled_methods == num_methods) { - type_ = kOatClassAllCompiled; - } else { - type_ = kOatClassSomeCompiled; - } + method_offsets_.resize(compiled_methods_with_code); + method_headers_.resize(compiled_methods_with_code); - status_ = status; - method_offsets_.resize(num_non_null_compiled_methods); - method_headers_.resize(num_non_null_compiled_methods); - - uint32_t oat_method_offsets_offset_from_oat_class = sizeof(type_) + sizeof(status_); - if (type_ == kOatClassSomeCompiled) { + uint32_t oat_method_offsets_offset_from_oat_class = OatClassHeader::SizeOf(); + // We only create this instance if there are at least some compiled. + if (oat_class_type == kOatClassSomeCompiled) { method_bitmap_.reset(new BitVector(num_methods, false, Allocator::GetMallocAllocator())); method_bitmap_size_ = method_bitmap_->GetSizeOf(); oat_method_offsets_offset_from_oat_class += sizeof(method_bitmap_size_); @@ -3084,43 +3441,27 @@ OatWriter::OatClass::OatClass(size_t offset, for (size_t i = 0; i < num_methods; i++) { CompiledMethod* compiled_method = compiled_methods_[i]; - if (compiled_method == nullptr) { - oat_method_offsets_offsets_from_oat_class_[i] = 0; - } else { + if (HasCompiledCode(compiled_method)) { oat_method_offsets_offsets_from_oat_class_[i] = oat_method_offsets_offset_from_oat_class; oat_method_offsets_offset_from_oat_class += sizeof(OatMethodOffsets); - if (type_ == kOatClassSomeCompiled) { + if (oat_class_type == kOatClassSomeCompiled) { method_bitmap_->SetBit(i); } + } else { + oat_method_offsets_offsets_from_oat_class_[i] = 0; } } } -size_t OatWriter::OatClass::GetOatMethodOffsetsOffsetFromOatHeader( - size_t class_def_method_index_) const { - uint32_t method_offset = GetOatMethodOffsetsOffsetFromOatClass(class_def_method_index_); - if (method_offset == 0) { - return 0; - } - return offset_ + method_offset; -} - -size_t OatWriter::OatClass::GetOatMethodOffsetsOffsetFromOatClass( - size_t class_def_method_index_) const { - return oat_method_offsets_offsets_from_oat_class_[class_def_method_index_]; -} - size_t OatWriter::OatClass::SizeOf() const { - return sizeof(status_) - + sizeof(type_) - + ((method_bitmap_size_ == 0) ? 0 : sizeof(method_bitmap_size_)) + return ((method_bitmap_size_ == 0) ? 0 : sizeof(method_bitmap_size_)) + method_bitmap_size_ + (sizeof(method_offsets_[0]) * method_offsets_.size()); } -bool OatWriter::OatClass::Write(OatWriter* oat_writer, - OutputStream* out, - const size_t file_offset) const { +bool OatWriter::OatClassHeader::Write(OatWriter* oat_writer, + OutputStream* out, + const size_t file_offset) const { DCHECK_OFFSET_(); if (!out->WriteFully(&status_, sizeof(status_))) { PLOG(ERROR) << "Failed to write class status to " << out->GetLocation(); @@ -3133,9 +3474,11 @@ bool OatWriter::OatClass::Write(OatWriter* oat_writer, return false; } oat_writer->size_oat_class_type_ += sizeof(type_); + return true; +} +bool OatWriter::OatClass::Write(OatWriter* oat_writer, OutputStream* out) const { if (method_bitmap_size_ != 0) { - CHECK_EQ(kOatClassSomeCompiled, type_); if (!out->WriteFully(&method_bitmap_size_, sizeof(method_bitmap_size_))) { PLOG(ERROR) << "Failed to write method bitmap size to " << out->GetLocation(); return false; diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h index 66b70ade2e..8db00f76d0 100644 --- a/compiler/oat_writer.h +++ b/compiler/oat_writer.h @@ -60,11 +60,6 @@ namespace verifier { // OatHeader variable length with count of D OatDexFiles // -// OatDexFile[0] one variable sized OatDexFile with offsets to Dex and OatClasses -// OatDexFile[1] -// ... -// OatDexFile[D] -// // TypeLookupTable[0] one descriptor to class def index hash table for each OatDexFile. // TypeLookupTable[1] // ... @@ -80,20 +75,25 @@ namespace verifier { // ... // OatClass[C] // -// GcMap one variable sized blob with GC map. -// GcMap GC maps are deduplicated. +// MethodBssMapping one variable sized MethodBssMapping for each dex file, optional. +// MethodBssMapping // ... -// GcMap +// MethodBssMapping // -// VmapTable one variable sized VmapTable blob (quick compiler only). +// VmapTable one variable sized VmapTable blob (CodeInfo or QuickeningInfo). // VmapTable VmapTables are deduplicated. // ... // VmapTable // -// MappingTable one variable sized blob with MappingTable (quick compiler only). -// MappingTable MappingTables are deduplicated. +// MethodInfo one variable sized blob with MethodInfo. +// MethodInfo MethodInfos are deduplicated. // ... -// MappingTable +// MethodInfo +// +// OatDexFile[0] one variable sized OatDexFile with offsets to Dex and OatClasses +// OatDexFile[1] +// ... +// OatDexFile[D] // // padding if necessary so that the following code will be page aligned // @@ -153,7 +153,7 @@ class OatWriter { const VdexFile& vdex_file, const char* location, CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault); - dchecked_vector<const char*> GetSourceLocations() const; + dchecked_vector<std::string> GetSourceLocations() const; // Write raw dex files to the vdex file, mmap the file and open the dex files from it. // Supporting data structures are written into the .rodata section of the oat file. @@ -217,6 +217,10 @@ class OatWriter { return bss_size_; } + size_t GetBssMethodsOffset() const { + return bss_methods_offset_; + } + size_t GetBssRootsOffset() const { return bss_roots_offset_; } @@ -235,12 +239,13 @@ class OatWriter { return ArrayRef<const debug::MethodDebugInfo>(method_info_); } - const CompilerDriver* GetCompilerDriver() { + const CompilerDriver* GetCompilerDriver() const { return compiler_driver_; } private: class DexFileSource; + class OatClassHeader; class OatClass; class OatDexFile; @@ -251,6 +256,7 @@ class OatWriter { // to actually write it. class DexMethodVisitor; class OatDexMethodVisitor; + class InitBssLayoutMethodVisitor; class InitOatClassesMethodVisitor; class InitCodeMethodVisitor; class InitMapMethodVisitor; @@ -260,6 +266,7 @@ class OatWriter { class WriteMapMethodVisitor; class WriteMethodInfoVisitor; class WriteQuickeningInfoMethodVisitor; + class WriteQuickeningIndicesMethodVisitor; // Visit all the methods in all the compiled dex files in their definition order // with a given DexMethodVisitor. @@ -295,29 +302,37 @@ class OatWriter { const InstructionSetFeatures* instruction_set_features, uint32_t num_dex_files, SafeMap<std::string, std::string>* key_value_store); - size_t InitOatDexFiles(size_t offset); + size_t InitClassOffsets(size_t offset); size_t InitOatClasses(size_t offset); size_t InitOatMaps(size_t offset); + size_t InitMethodBssMappings(size_t offset); + size_t InitOatDexFiles(size_t offset); size_t InitOatCode(size_t offset); size_t InitOatCodeDexFiles(size_t offset); void InitBssLayout(InstructionSet instruction_set); - bool WriteClassOffsets(OutputStream* out); - bool WriteClasses(OutputStream* out); - size_t WriteMaps(OutputStream* out, const size_t file_offset, size_t relative_offset); - size_t WriteCode(OutputStream* out, const size_t file_offset, size_t relative_offset); - size_t WriteCodeDexFiles(OutputStream* out, const size_t file_offset, size_t relative_offset); + size_t WriteClassOffsets(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteClasses(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteMaps(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteMethodBssMappings(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteOatDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteCode(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteCodeDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset); bool RecordOatDataOffset(OutputStream* out); bool ReadDexFileHeader(File* oat_file, OatDexFile* oat_dex_file); bool ValidateDexFileHeader(const uint8_t* raw_header, const char* location); - bool WriteOatDexFiles(OutputStream* oat_rodata); bool WriteTypeLookupTables(OutputStream* oat_rodata, const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files); + bool WriteDexLayoutSections(OutputStream* oat_rodata, + const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files); bool WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta); + bool WriteUpTo16BytesAlignment(OutputStream* out, uint32_t size, uint32_t* stat); void SetMultiOatRelativePatcherAdjustment(); void CloseSources(); + bool MayHaveCompiledMethods() const; + enum class WriteState { kAddingDexFileSources, kPrepareLayout, @@ -368,9 +383,20 @@ class OatWriter { // The size of the required .bss section holding the DexCache data and GC roots. size_t bss_size_; + // The offset of the methods in .bss section. + size_t bss_methods_offset_; + // The offset of the GC roots in .bss section. size_t bss_roots_offset_; + // Map for recording references to ArtMethod entries in .bss. + SafeMap<const DexFile*, BitVector> bss_method_entry_references_; + + // Map for allocating ArtMethod entries in .bss. Indexed by MethodReference for the target + // method in the dex file with the "method reference value comparator" for deduplication. + // The value is the target offset for patching, starting at `bss_start_ + bss_methods_offset_`. + SafeMap<MethodReference, size_t, MethodReferenceValueComparator> bss_method_entries_; + // Map for allocating Class entries in .bss. Indexed by TypeReference for the source // type in the dex file with the "type value comparator" for deduplication. The value // is the target offset for patching, starting at `bss_start_ + bss_roots_offset_`. @@ -381,10 +407,6 @@ class OatWriter { // is the target offset for patching, starting at `bss_start_ + bss_roots_offset_`. SafeMap<StringReference, size_t, StringReferenceValueComparator> bss_string_entries_; - // Offsets of the dex cache arrays for each app dex file. For the - // boot image, this information is provided by the ImageWriter. - SafeMap<const DexFile*, size_t> dex_cache_arrays_offsets_; // DexFiles not owned. - // Offset of the oat data from the start of the mmapped region of the elf file. size_t oat_data_offset_; @@ -394,6 +416,7 @@ class OatWriter { // data to write std::unique_ptr<OatHeader> oat_header_; dchecked_vector<OatDexFile> oat_dex_files_; + dchecked_vector<OatClassHeader> oat_class_headers_; dchecked_vector<OatClass> oat_classes_; std::unique_ptr<const std::vector<uint8_t>> jni_dlsym_lookup_; std::unique_ptr<const std::vector<uint8_t>> quick_generic_jni_trampoline_; @@ -434,6 +457,10 @@ class OatWriter { uint32_t size_oat_dex_file_offset_; uint32_t size_oat_dex_file_class_offsets_offset_; uint32_t size_oat_dex_file_lookup_table_offset_; + uint32_t size_oat_dex_file_dex_layout_sections_offset_; + uint32_t size_oat_dex_file_dex_layout_sections_; + uint32_t size_oat_dex_file_dex_layout_sections_alignment_; + uint32_t size_oat_dex_file_method_bss_mapping_offset_; uint32_t size_oat_lookup_table_alignment_; uint32_t size_oat_lookup_table_; uint32_t size_oat_class_offsets_alignment_; @@ -442,6 +469,7 @@ class OatWriter { uint32_t size_oat_class_status_; uint32_t size_oat_class_method_bitmaps_; uint32_t size_oat_class_method_offsets_; + uint32_t size_method_bss_mappings_; // The helper for processing relative patches is external so that we can patch across oat files. linker::MultiOatRelativePatcher* relative_patcher_; diff --git a/compiler/optimizing/block_builder.cc b/compiler/optimizing/block_builder.cc index 1e75f10ebe..fe7ecd1ae1 100644 --- a/compiler/optimizing/block_builder.cc +++ b/compiler/optimizing/block_builder.cc @@ -17,6 +17,7 @@ #include "block_builder.h" #include "bytecode_utils.h" +#include "quicken_info.h" namespace art { @@ -121,13 +122,18 @@ void HBasicBlockBuilder::ConnectBasicBlocks() { HBasicBlock* block = graph_->GetEntryBlock(); graph_->AddBlock(block); + size_t quicken_index = 0; bool is_throwing_block = false; + // Calculate the qucikening index here instead of CreateBranchTargets since it's easier to + // calculate in dex_pc order. for (CodeItemIterator it(code_item_); !it.Done(); it.Advance()) { uint32_t dex_pc = it.CurrentDexPc(); // Check if this dex_pc address starts a new basic block. HBasicBlock* next_block = GetBlockAt(dex_pc); if (next_block != nullptr) { + // We only need quicken index entries for basic block boundaries. + quicken_index_for_dex_pc_.Put(dex_pc, quicken_index); if (block != nullptr) { // Last instruction did not end its basic block but a new one starts here. // It must have been a block falling through into the next one. @@ -137,6 +143,10 @@ void HBasicBlockBuilder::ConnectBasicBlocks() { is_throwing_block = false; graph_->AddBlock(block); } + // Make sure to increment this before the continues. + if (QuickenInfoTable::NeedsIndexForInstruction(&it.CurrentInstruction())) { + ++quicken_index; + } if (block == nullptr) { // Ignore dead code. @@ -371,4 +381,8 @@ bool HBasicBlockBuilder::Build() { return true; } +size_t HBasicBlockBuilder::GetQuickenIndex(uint32_t dex_pc) const { + return quicken_index_for_dex_pc_.Get(dex_pc); +} + } // namespace art diff --git a/compiler/optimizing/block_builder.h b/compiler/optimizing/block_builder.h index 1be0b4ce98..6adce815f4 100644 --- a/compiler/optimizing/block_builder.h +++ b/compiler/optimizing/block_builder.h @@ -37,7 +37,8 @@ class HBasicBlockBuilder : public ValueObject { nullptr, arena_->Adapter(kArenaAllocGraphBuilder)), throwing_blocks_(kDefaultNumberOfThrowingBlocks, arena_->Adapter(kArenaAllocGraphBuilder)), - number_of_branches_(0u) {} + number_of_branches_(0u), + quicken_index_for_dex_pc_(std::less<uint32_t>(), arena_->Adapter()) {} // Creates basic blocks in `graph_` at branch target dex_pc positions of the // `code_item_`. Blocks are connected but left unpopulated with instructions. @@ -48,6 +49,8 @@ class HBasicBlockBuilder : public ValueObject { size_t GetNumberOfBranches() const { return number_of_branches_; } HBasicBlock* GetBlockAt(uint32_t dex_pc) const { return branch_targets_[dex_pc]; } + size_t GetQuickenIndex(uint32_t dex_pc) const; + private: // Creates a basic block starting at given `dex_pc`. HBasicBlock* MaybeCreateBlockAt(uint32_t dex_pc); @@ -78,6 +81,9 @@ class HBasicBlockBuilder : public ValueObject { ArenaVector<HBasicBlock*> throwing_blocks_; size_t number_of_branches_; + // A table to quickly find the quicken index for the first instruction of a basic block. + ArenaSafeMap<uint32_t, uint32_t> quicken_index_for_dex_pc_; + static constexpr size_t kDefaultNumberOfThrowingBlocks = 2u; DISALLOW_COPY_AND_ASSIGN(HBasicBlockBuilder); diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index c918ee6687..2872cf7458 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -17,7 +17,6 @@ #include "code_generator.h" #ifdef ART_ENABLE_CODEGEN_arm -#include "code_generator_arm.h" #include "code_generator_arm_vixl.h" #endif @@ -41,6 +40,8 @@ #include "code_generator_mips64.h" #endif +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "bytecode_utils.h" #include "class_linker.h" #include "compiled_method.h" @@ -337,7 +338,7 @@ void CodeGenerator::CreateCommonInvokeLocationSummary( case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: locations->AddTemp(visitor->GetMethodLocation()); locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister()); break; @@ -350,6 +351,34 @@ void CodeGenerator::CreateCommonInvokeLocationSummary( } } +void CodeGenerator::GenerateInvokeStaticOrDirectRuntimeCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { + MoveConstant(temp, invoke->GetDexMethodIndex()); + + // The access check is unnecessary but we do not want to introduce + // extra entrypoints for the codegens that do not support some + // invoke type and fall back to the runtime call. + + // Initialize to anything to silent compiler warnings. + QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck; + switch (invoke->GetInvokeType()) { + case kStatic: + entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck; + break; + case kDirect: + entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck; + break; + case kSuper: + entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck; + break; + case kVirtual: + case kInterface: + LOG(FATAL) << "Unexpected invoke type: " << invoke->GetInvokeType(); + UNREACHABLE(); + } + + InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), slow_path); +} void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) { MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex()); @@ -597,19 +626,11 @@ std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph, #ifdef ART_ENABLE_CODEGEN_arm case kArm: case kThumb2: { - if (kArmUseVIXL32) { - return std::unique_ptr<CodeGenerator>( - new (arena) arm::CodeGeneratorARMVIXL(graph, - *isa_features.AsArmInstructionSetFeatures(), - compiler_options, - stats)); - } else { - return std::unique_ptr<CodeGenerator>( - new (arena) arm::CodeGeneratorARM(graph, - *isa_features.AsArmInstructionSetFeatures(), - compiler_options, - stats)); - } + return std::unique_ptr<CodeGenerator>( + new (arena) arm::CodeGeneratorARMVIXL(graph, + *isa_features.AsArmInstructionSetFeatures(), + compiler_options, + stats)); } #endif #ifdef ART_ENABLE_CODEGEN_arm64 diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index c9ba5c3357..73202b4fd1 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -404,17 +404,6 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { // accessing the String's `value` field in String intrinsics. static uint32_t GetArrayDataOffset(HArrayGet* array_get); - // Return the entry point offset for ReadBarrierMarkRegX, where X is `reg`. - template <PointerSize pointer_size> - static int32_t GetReadBarrierMarkEntryPointsOffset(size_t reg) { - // The entry point list defines 30 ReadBarrierMarkRegX entry points. - DCHECK_LT(reg, 30u); - // The ReadBarrierMarkRegX entry points are ordered by increasing - // register number in Thread::tls_Ptr_.quick_entrypoints. - return QUICK_ENTRYPOINT_OFFSET(pointer_size, pReadBarrierMarkReg00).Int32Value() - + static_cast<size_t>(pointer_size) * reg; - } - void EmitParallelMoves(Location from1, Location to1, Primitive::Type type1, @@ -496,6 +485,8 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { static void CreateCommonInvokeLocationSummary( HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor); + void GenerateInvokeStaticOrDirectRuntimeCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path); void GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke); void GenerateInvokePolymorphicCall(HInvokePolymorphic* invoke); @@ -564,9 +555,11 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { HInvokeStaticOrDirect* invoke) = 0; // Generate a call to a static or direct method. - virtual void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) = 0; + virtual void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) = 0; // Generate a call to a virtual method. - virtual void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) = 0; + virtual void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) = 0; // Copy the result of a call into the given target. virtual void MoveFromReturnRegister(Location trg, Primitive::Type type) = 0; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc deleted file mode 100644 index 914ae177c4..0000000000 --- a/compiler/optimizing/code_generator_arm.cc +++ /dev/null @@ -1,9435 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "code_generator_arm.h" - -#include "arch/arm/asm_support_arm.h" -#include "arch/arm/instruction_set_features_arm.h" -#include "art_method.h" -#include "code_generator_utils.h" -#include "common_arm.h" -#include "compiled_method.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "gc/accounting/card_table.h" -#include "intrinsics.h" -#include "intrinsics_arm.h" -#include "linker/arm/relative_patcher_thumb2.h" -#include "mirror/array-inl.h" -#include "mirror/class-inl.h" -#include "thread.h" -#include "utils/arm/assembler_arm.h" -#include "utils/arm/managed_register_arm.h" -#include "utils/assembler.h" -#include "utils/stack_checks.h" - -namespace art { - -template<class MirrorType> -class GcRoot; - -namespace arm { - -static bool ExpectedPairLayout(Location location) { - // We expected this for both core and fpu register pairs. - return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); -} - -static constexpr int kCurrentMethodStackOffset = 0; -static constexpr Register kMethodRegisterArgument = R0; - -static constexpr Register kCoreAlwaysSpillRegister = R5; -static constexpr Register kCoreCalleeSaves[] = - { R5, R6, R7, R8, R10, R11, LR }; -static constexpr SRegister kFpuCalleeSaves[] = - { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; - -// D31 cannot be split into two S registers, and the register allocator only works on -// S registers. Therefore there is no need to block it. -static constexpr DRegister DTMP = D31; - -static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; - -// Reference load (except object array loads) is using LDR Rt, [Rn, #offset] which can handle -// offset < 4KiB. For offsets >= 4KiB, the load shall be emitted as two or more instructions. -// For the Baker read barrier implementation using link-generated thunks we need to split -// the offset explicitly. -constexpr uint32_t kReferenceLoadMinFarOffset = 4 * KB; - -// Flags controlling the use of link-time generated thunks for Baker read barriers. -constexpr bool kBakerReadBarrierLinkTimeThunksEnableForFields = true; -constexpr bool kBakerReadBarrierLinkTimeThunksEnableForArrays = true; -constexpr bool kBakerReadBarrierLinkTimeThunksEnableForGcRoots = true; - -// The reserved entrypoint register for link-time generated thunks. -const Register kBakerCcEntrypointRegister = R4; - -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT -#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() - -static inline void CheckLastTempIsBakerCcEntrypointRegister(HInstruction* instruction) { - DCHECK_EQ(static_cast<uint32_t>(kBakerCcEntrypointRegister), - linker::Thumb2RelativePatcher::kBakerCcEntrypointRegister); - DCHECK_NE(instruction->GetLocations()->GetTempCount(), 0u); - DCHECK_EQ(kBakerCcEntrypointRegister, - instruction->GetLocations()->GetTemp( - instruction->GetLocations()->GetTempCount() - 1u).AsRegister<Register>()); -} - -static inline void EmitPlaceholderBne(CodeGeneratorARM* codegen, Label* bne_label) { - ScopedForce32Bit force_32bit(down_cast<Thumb2Assembler*>(codegen->GetAssembler())); - __ BindTrackedLabel(bne_label); - Label placeholder_label; - __ b(&placeholder_label, NE); // Placeholder, patched at link-time. - __ Bind(&placeholder_label); -} - -static inline bool CanEmitNarrowLdr(Register rt, Register rn, uint32_t offset) { - return ArmAssembler::IsLowRegister(rt) && ArmAssembler::IsLowRegister(rn) && offset < 32u; -} - -static constexpr int kRegListThreshold = 4; - -// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, -// for each live D registers they treat two corresponding S registers as live ones. -// -// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build -// from a list of contiguous S registers a list of contiguous D registers (processing first/last -// S registers corner cases) and save/restore this new list treating them as D registers. -// - decreasing code size -// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is -// restored and then used in regular non SlowPath code as D register. -// -// For the following example (v means the S register is live): -// D names: | D0 | D1 | D2 | D4 | ... -// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... -// Live? | | v | v | v | v | v | v | | ... -// -// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed -// as D registers. -static size_t SaveContiguousSRegisterList(size_t first, - size_t last, - CodeGenerator* codegen, - size_t stack_offset) { - DCHECK_LE(first, last); - if ((first == last) && (first == 0)) { - stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); - return stack_offset; - } - if (first % 2 == 1) { - stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); - } - - bool save_last = false; - if (last % 2 == 0) { - save_last = true; - --last; - } - - if (first < last) { - DRegister d_reg = static_cast<DRegister>(first / 2); - DCHECK_EQ((last - first + 1) % 2, 0u); - size_t number_of_d_regs = (last - first + 1) / 2; - - if (number_of_d_regs == 1) { - __ StoreDToOffset(d_reg, SP, stack_offset); - } else if (number_of_d_regs > 1) { - __ add(IP, SP, ShifterOperand(stack_offset)); - __ vstmiad(IP, d_reg, number_of_d_regs); - } - stack_offset += number_of_d_regs * kArmWordSize * 2; - } - - if (save_last) { - stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); - } - - return stack_offset; -} - -static size_t RestoreContiguousSRegisterList(size_t first, - size_t last, - CodeGenerator* codegen, - size_t stack_offset) { - DCHECK_LE(first, last); - if ((first == last) && (first == 0)) { - stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); - return stack_offset; - } - if (first % 2 == 1) { - stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); - } - - bool restore_last = false; - if (last % 2 == 0) { - restore_last = true; - --last; - } - - if (first < last) { - DRegister d_reg = static_cast<DRegister>(first / 2); - DCHECK_EQ((last - first + 1) % 2, 0u); - size_t number_of_d_regs = (last - first + 1) / 2; - if (number_of_d_regs == 1) { - __ LoadDFromOffset(d_reg, SP, stack_offset); - } else if (number_of_d_regs > 1) { - __ add(IP, SP, ShifterOperand(stack_offset)); - __ vldmiad(IP, d_reg, number_of_d_regs); - } - stack_offset += number_of_d_regs * kArmWordSize * 2; - } - - if (restore_last) { - stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); - } - - return stack_offset; -} - -void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { - size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); - size_t orig_offset = stack_offset; - - const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); - for (uint32_t i : LowToHighBits(core_spills)) { - // If the register holds an object, update the stack mask. - if (locations->RegisterContainsObject(i)) { - locations->SetStackBit(stack_offset / kVRegSize); - } - DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); - DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); - saved_core_stack_offsets_[i] = stack_offset; - stack_offset += kArmWordSize; - } - - int reg_num = POPCOUNT(core_spills); - if (reg_num != 0) { - if (reg_num > kRegListThreshold) { - __ StoreList(RegList(core_spills), orig_offset); - } else { - stack_offset = orig_offset; - for (uint32_t i : LowToHighBits(core_spills)) { - stack_offset += codegen->SaveCoreRegister(stack_offset, i); - } - } - } - - uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); - orig_offset = stack_offset; - for (uint32_t i : LowToHighBits(fp_spills)) { - DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); - saved_fpu_stack_offsets_[i] = stack_offset; - stack_offset += kArmWordSize; - } - - stack_offset = orig_offset; - while (fp_spills != 0u) { - uint32_t begin = CTZ(fp_spills); - uint32_t tmp = fp_spills + (1u << begin); - fp_spills &= tmp; // Clear the contiguous range of 1s. - uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. - stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); - } - DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); -} - -void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { - size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); - size_t orig_offset = stack_offset; - - const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); - for (uint32_t i : LowToHighBits(core_spills)) { - DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); - DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); - stack_offset += kArmWordSize; - } - - int reg_num = POPCOUNT(core_spills); - if (reg_num != 0) { - if (reg_num > kRegListThreshold) { - __ LoadList(RegList(core_spills), orig_offset); - } else { - stack_offset = orig_offset; - for (uint32_t i : LowToHighBits(core_spills)) { - stack_offset += codegen->RestoreCoreRegister(stack_offset, i); - } - } - } - - uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); - while (fp_spills != 0u) { - uint32_t begin = CTZ(fp_spills); - uint32_t tmp = fp_spills + (1u << begin); - fp_spills &= tmp; // Clear the contiguous range of 1s. - uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. - stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); - } - DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); -} - -class NullCheckSlowPathARM : public SlowPathCodeARM { - public: - explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - if (instruction_->CanThrowIntoCatchBlock()) { - // Live registers will be restored in the catch block if caught. - SaveLiveRegisters(codegen, instruction_->GetLocations()); - } - arm_codegen->InvokeRuntime(kQuickThrowNullPointer, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); - } - - bool IsFatal() const OVERRIDE { return true; } - - const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); -}; - -class DivZeroCheckSlowPathARM : public SlowPathCodeARM { - public: - explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); - } - - bool IsFatal() const OVERRIDE { return true; } - - const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); -}; - -class SuspendCheckSlowPathARM : public SlowPathCodeARM { - public: - SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) - : SlowPathCodeARM(instruction), successor_(successor) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickTestSuspend, void, void>(); - if (successor_ == nullptr) { - __ b(GetReturnLabel()); - } else { - __ b(arm_codegen->GetLabelOf(successor_)); - } - } - - Label* GetReturnLabel() { - DCHECK(successor_ == nullptr); - return &return_label_; - } - - HBasicBlock* GetSuccessor() const { - return successor_; - } - - const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } - - private: - // If not null, the block to branch to after the suspend check. - HBasicBlock* const successor_; - - // If `successor_` is null, the label to branch to after the suspend check. - Label return_label_; - - DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); -}; - -class BoundsCheckSlowPathARM : public SlowPathCodeARM { - public: - explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) - : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - LocationSummary* locations = instruction_->GetLocations(); - - __ Bind(GetEntryLabel()); - if (instruction_->CanThrowIntoCatchBlock()) { - // Live registers will be restored in the catch block if caught. - SaveLiveRegisters(codegen, instruction_->GetLocations()); - } - // We're moving two locations to locations that could overlap, so we need a parallel - // move resolver. - InvokeRuntimeCallingConvention calling_convention; - codegen->EmitParallelMoves( - locations->InAt(0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimInt, - locations->InAt(1), - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimInt); - QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() - ? kQuickThrowStringBounds - : kQuickThrowArrayBounds; - arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); - CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); - } - - bool IsFatal() const OVERRIDE { return true; } - - const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); -}; - -class LoadClassSlowPathARM : public SlowPathCodeARM { - public: - LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) - : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { - DCHECK(at->IsLoadClass() || at->IsClinitCheck()); - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Location out = locations->Out(); - constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - bool is_load_class_bss_entry = - (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); - Register entry_address = kNoRegister; - if (is_load_class_bss_entry && call_saves_everything_except_r0) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call. - bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); - entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; - DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); - if (temp_is_r0) { - __ mov(entry_address, ShifterOperand(temp)); - } - } - dex::TypeIndex type_index = cls_->GetTypeIndex(); - __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); - QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage - : kQuickInitializeType; - arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); - if (do_clinit_) { - CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); - } else { - CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); - } - - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - if (is_load_class_bss_entry) { - if (call_saves_everything_except_r0) { - // The class entry address was preserved in `entry_address` thanks to kSaveEverything. - __ str(R0, Address(entry_address)); - } else { - // For non-Baker read barrier, we need to re-calculate the address of the string entry. - Register temp = IP; - CodeGeneratorARM::PcRelativePatchInfo* labels = - arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - __ str(R0, Address(temp)); - } - } - // Move the class to the desired location. - if (out.IsValid()) { - DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); - arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); - } - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } - - private: - // The class this slow path will load. - HLoadClass* const cls_; - - // The dex PC of `at_`. - const uint32_t dex_pc_; - - // Whether to initialize the class. - const bool do_clinit_; - - DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); -}; - -class LoadStringSlowPathARM : public SlowPathCodeARM { - public: - explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - DCHECK(instruction_->IsLoadString()); - DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); - HLoadString* load = instruction_->AsLoadString(); - const dex::StringIndex string_index = load->GetStringIndex(); - Register out = locations->Out().AsRegister<Register>(); - constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call. - Register entry_address = kNoRegister; - if (call_saves_everything_except_r0) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); - entry_address = temp_is_r0 ? out : temp; - DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); - if (temp_is_r0) { - __ mov(entry_address, ShifterOperand(temp)); - } - } - - __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); - arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); - - // Store the resolved String to the .bss entry. - if (call_saves_everything_except_r0) { - // The string entry address was preserved in `entry_address` thanks to kSaveEverything. - __ str(R0, Address(entry_address)); - } else { - // For non-Baker read barrier, we need to re-calculate the address of the string entry. - Register temp = IP; - CodeGeneratorARM::PcRelativePatchInfo* labels = - arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - __ str(R0, Address(temp)); - } - - arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); - RestoreLiveRegisters(codegen, locations); - - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); -}; - -class TypeCheckSlowPathARM : public SlowPathCodeARM { - public: - TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) - : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(instruction_->IsCheckCast() - || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - - if (!is_fatal_) { - SaveLiveRegisters(codegen, locations); - } - - // We're moving two locations to locations that could overlap, so we need a parallel - // move resolver. - InvokeRuntimeCallingConvention calling_convention; - codegen->EmitParallelMoves(locations->InAt(0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimNot, - locations->InAt(1), - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimNot); - if (instruction_->IsInstanceOf()) { - arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); - arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); - } else { - DCHECK(instruction_->IsCheckCast()); - arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); - } - - if (!is_fatal_) { - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - } - - const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } - - bool IsFatal() const OVERRIDE { return is_fatal_; } - - private: - const bool is_fatal_; - - DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); -}; - -class DeoptimizationSlowPathARM : public SlowPathCodeARM { - public: - explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) - : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - LocationSummary* locations = instruction_->GetLocations(); - SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; - __ LoadImmediate(calling_convention.GetRegisterAt(0), - static_cast<uint32_t>(instruction_->AsDeoptimize()->GetDeoptimizationKind())); - arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickDeoptimize, void, DeoptimizationKind>(); - } - - const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); -}; - -class ArraySetSlowPathARM : public SlowPathCodeARM { - public: - explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - HParallelMove parallel_move(codegen->GetGraph()->GetArena()); - parallel_move.AddMove( - locations->InAt(0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimNot, - nullptr); - parallel_move.AddMove( - locations->InAt(1), - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimInt, - nullptr); - parallel_move.AddMove( - locations->InAt(2), - Location::RegisterLocation(calling_convention.GetRegisterAt(2)), - Primitive::kPrimNot, - nullptr); - codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); -}; - -// Abstract base class for read barrier slow paths marking a reference -// `ref`. -// -// Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. -class ReadBarrierMarkSlowPathBaseARM : public SlowPathCodeARM { - protected: - ReadBarrierMarkSlowPathBaseARM(HInstruction* instruction, Location ref, Location entrypoint) - : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { - DCHECK(kEmitCompilerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM"; } - - // Generate assembly code calling the read barrier marking runtime - // entry point (ReadBarrierMarkRegX). - void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) { - Register ref_reg = ref_.AsRegister<Register>(); - - // No need to save live registers; it's taken care of by the - // entrypoint. Also, there is no need to update the stack mask, - // as this runtime call will not trigger a garbage collection. - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - DCHECK_NE(ref_reg, SP); - DCHECK_NE(ref_reg, LR); - DCHECK_NE(ref_reg, PC); - // IP is used internally by the ReadBarrierMarkRegX entry point - // as a temporary, it cannot be the entry point's input/output. - DCHECK_NE(ref_reg, IP); - DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; - // "Compact" slow path, saving two moves. - // - // Instead of using the standard runtime calling convention (input - // and output in R0): - // - // R0 <- ref - // R0 <- ReadBarrierMark(R0) - // ref <- R0 - // - // we just use rX (the register containing `ref`) as input and output - // of a dedicated entrypoint: - // - // rX <- ReadBarrierMarkRegX(rX) - // - if (entrypoint_.IsValid()) { - arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); - __ blx(entrypoint_.AsRegister<Register>()); - } else { - // Entrypoint is not already loaded, load from the thread. - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); - // This runtime call does not require a stack map. - arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); - } - } - - // The location (register) of the marked object reference. - const Location ref_; - - // The location of the entrypoint if it is already loaded. - const Location entrypoint_; - - private: - DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM); -}; - -// Slow path marking an object reference `ref` during a read -// barrier. The field `obj.field` in the object `obj` holding this -// reference does not get updated by this slow path after marking. -// -// This means that after the execution of this slow path, `ref` will -// always be up-to-date, but `obj.field` may not; i.e., after the -// flip, `ref` will be a to-space reference, but `obj.field` will -// probably still be a from-space reference (unless it gets updated by -// another thread, or if another thread installed another object -// reference (different from `ref`) in `obj.field`). -// -// If `entrypoint` is a valid location it is assumed to already be -// holding the entrypoint. The case where the entrypoint is passed in -// is when the decision to mark is based on whether the GC is marking. -class ReadBarrierMarkSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { - public: - ReadBarrierMarkSlowPathARM(HInstruction* instruction, - Location ref, - Location entrypoint = Location::NoLocation()) - : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint) { - DCHECK(kEmitCompilerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(locations->CanCall()); - if (kIsDebugBuild) { - Register ref_reg = ref_.AsRegister<Register>(); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; - } - DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) - << "Unexpected instruction in read barrier marking slow path: " - << instruction_->DebugName(); - - __ Bind(GetEntryLabel()); - GenerateReadBarrierMarkRuntimeCall(codegen); - __ b(GetExitLabel()); - } - - private: - DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); -}; - -// Slow path loading `obj`'s lock word, loading a reference from -// object `*(obj + offset + (index << scale_factor))` into `ref`, and -// marking `ref` if `obj` is gray according to the lock word (Baker -// read barrier). The field `obj.field` in the object `obj` holding -// this reference does not get updated by this slow path after marking -// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM -// below for that). -// -// This means that after the execution of this slow path, `ref` will -// always be up-to-date, but `obj.field` may not; i.e., after the -// flip, `ref` will be a to-space reference, but `obj.field` will -// probably still be a from-space reference (unless it gets updated by -// another thread, or if another thread installed another object -// reference (different from `ref`) in `obj.field`). -// -// Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. -class LoadReferenceWithBakerReadBarrierSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { - public: - LoadReferenceWithBakerReadBarrierSlowPathARM(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check, - Register temp, - Location entrypoint) - : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), - obj_(obj), - offset_(offset), - index_(index), - scale_factor_(scale_factor), - needs_null_check_(needs_null_check), - temp_(temp) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { - return "LoadReferenceWithBakerReadBarrierSlowPathARM"; - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Register ref_reg = ref_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; - DCHECK_NE(ref_reg, temp_); - DCHECK(instruction_->IsInstanceFieldGet() || - instruction_->IsStaticFieldGet() || - instruction_->IsArrayGet() || - instruction_->IsArraySet() || - instruction_->IsInstanceOf() || - instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || - (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) - << "Unexpected instruction in read barrier marking slow path: " - << instruction_->DebugName(); - // The read barrier instrumentation of object ArrayGet - // instructions does not support the HIntermediateAddress - // instruction. - DCHECK(!(instruction_->IsArrayGet() && - instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); - - __ Bind(GetEntryLabel()); - - // When using MaybeGenerateReadBarrierSlow, the read barrier call is - // inserted after the original load. However, in fast path based - // Baker's read barriers, we need to perform the load of - // mirror::Object::monitor_ *before* the original reference load. - // This load-load ordering is required by the read barrier. - // The slow path (for Baker's algorithm) should look like: - // - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // } - // - // Note: the original implementation in ReadBarrier::Barrier is - // slightly more complex as it performs additional checks that we do - // not do here for performance reasons. - - // /* int32_t */ monitor = obj->monitor_ - uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); - __ LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset); - if (needs_null_check_) { - codegen->MaybeRecordImplicitNullCheck(instruction_); - } - // /* LockWord */ lock_word = LockWord(monitor) - static_assert(sizeof(LockWord) == sizeof(int32_t), - "art::LockWord and int32_t have different sizes."); - - // Introduce a dependency on the lock_word including the rb_state, - // which shall prevent load-load reordering without using - // a memory barrier (which would be more expensive). - // `obj` is unchanged by this operation, but its value now depends - // on `temp`. - __ add(obj_, obj_, ShifterOperand(temp_, LSR, 32)); - - // The actual reference load. - // A possible implicit null check has already been handled above. - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->GenerateRawReferenceLoad( - instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); - - // Mark the object `ref` when `obj` is gray. - // - // if (rb_state == ReadBarrier::GrayState()) - // ref = ReadBarrier::Mark(ref); - // - // Given the numeric representation, it's enough to check the low bit of the - // rb_state. We do that by shifting the bit out of the lock word with LSRS - // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1); - __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. - GenerateReadBarrierMarkRuntimeCall(codegen); - - __ b(GetExitLabel()); - } - - private: - // The register containing the object holding the marked object reference field. - Register obj_; - // The offset, index and scale factor to access the reference in `obj_`. - uint32_t offset_; - Location index_; - ScaleFactor scale_factor_; - // Is a null check required? - bool needs_null_check_; - // A temporary register used to hold the lock word of `obj_`. - Register temp_; - - DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM); -}; - -// Slow path loading `obj`'s lock word, loading a reference from -// object `*(obj + offset + (index << scale_factor))` into `ref`, and -// marking `ref` if `obj` is gray according to the lock word (Baker -// read barrier). If needed, this slow path also atomically updates -// the field `obj.field` in the object `obj` holding this reference -// after marking (contrary to -// LoadReferenceWithBakerReadBarrierSlowPathARM above, which never -// tries to update `obj.field`). -// -// This means that after the execution of this slow path, both `ref` -// and `obj.field` will be up-to-date; i.e., after the flip, both will -// hold the same to-space reference (unless another thread installed -// another object reference (different from `ref`) in `obj.field`). -// -// Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. -class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM - : public ReadBarrierMarkSlowPathBaseARM { - public: - LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check, - Register temp1, - Register temp2, - Location entrypoint) - : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), - obj_(obj), - offset_(offset), - index_(index), - scale_factor_(scale_factor), - needs_null_check_(needs_null_check), - temp1_(temp1), - temp2_(temp2) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { - return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM"; - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Register ref_reg = ref_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; - DCHECK_NE(ref_reg, temp1_); - - // This slow path is only used by the UnsafeCASObject intrinsic at the moment. - DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) - << "Unexpected instruction in read barrier marking and field updating slow path: " - << instruction_->DebugName(); - DCHECK(instruction_->GetLocations()->Intrinsified()); - DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); - DCHECK_EQ(offset_, 0u); - DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1); - // The location of the offset of the marked reference field within `obj_`. - Location field_offset = index_; - DCHECK(field_offset.IsRegisterPair()) << field_offset; - - __ Bind(GetEntryLabel()); - - // The implementation is similar to LoadReferenceWithBakerReadBarrierSlowPathARM's: - // - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // old_ref = ref; - // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // compareAndSwapObject(obj, field_offset, old_ref, ref); - // } - - // /* int32_t */ monitor = obj->monitor_ - uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); - __ LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset); - if (needs_null_check_) { - codegen->MaybeRecordImplicitNullCheck(instruction_); - } - // /* LockWord */ lock_word = LockWord(monitor) - static_assert(sizeof(LockWord) == sizeof(int32_t), - "art::LockWord and int32_t have different sizes."); - - // Introduce a dependency on the lock_word including the rb_state, - // which shall prevent load-load reordering without using - // a memory barrier (which would be more expensive). - // `obj` is unchanged by this operation, but its value now depends - // on `temp1`. - __ add(obj_, obj_, ShifterOperand(temp1_, LSR, 32)); - - // The actual reference load. - // A possible implicit null check has already been handled above. - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->GenerateRawReferenceLoad( - instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); - - // Mark the object `ref` when `obj` is gray. - // - // if (rb_state == ReadBarrier::GrayState()) - // ref = ReadBarrier::Mark(ref); - // - // Given the numeric representation, it's enough to check the low bit of the - // rb_state. We do that by shifting the bit out of the lock word with LSRS - // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1); - __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. - - // Save the old value of the reference before marking it. - // Note that we cannot use IP to save the old reference, as IP is - // used internally by the ReadBarrierMarkRegX entry point, and we - // need the old reference after the call to that entry point. - DCHECK_NE(temp1_, IP); - __ Mov(temp1_, ref_reg); - - GenerateReadBarrierMarkRuntimeCall(codegen); - - // If the new reference is different from the old reference, - // update the field in the holder (`*(obj_ + field_offset)`). - // - // Note that this field could also hold a different object, if - // another thread had concurrently changed it. In that case, the - // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set - // (CAS) operation below would abort the CAS, leaving the field - // as-is. - __ cmp(temp1_, ShifterOperand(ref_reg)); - __ b(GetExitLabel(), EQ); - - // Update the the holder's field atomically. This may fail if - // mutator updates before us, but it's OK. This is achieved - // using a strong compare-and-set (CAS) operation with relaxed - // memory synchronization ordering, where the expected value is - // the old reference and the desired value is the new reference. - - // Convenience aliases. - Register base = obj_; - // The UnsafeCASObject intrinsic uses a register pair as field - // offset ("long offset"), of which only the low part contains - // data. - Register offset = field_offset.AsRegisterPairLow<Register>(); - Register expected = temp1_; - Register value = ref_reg; - Register tmp_ptr = IP; // Pointer to actual memory. - Register tmp = temp2_; // Value in memory. - - __ add(tmp_ptr, base, ShifterOperand(offset)); - - if (kPoisonHeapReferences) { - __ PoisonHeapReference(expected); - if (value == expected) { - // Do not poison `value`, as it is the same register as - // `expected`, which has just been poisoned. - } else { - __ PoisonHeapReference(value); - } - } - - // do { - // tmp = [r_ptr] - expected; - // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); - - Label loop_head, exit_loop; - __ Bind(&loop_head); - - __ ldrex(tmp, tmp_ptr); - - __ subs(tmp, tmp, ShifterOperand(expected)); - - __ it(NE); - __ clrex(NE); - - __ b(&exit_loop, NE); - - __ strex(tmp, value, tmp_ptr); - __ cmp(tmp, ShifterOperand(1)); - __ b(&loop_head, EQ); - - __ Bind(&exit_loop); - - if (kPoisonHeapReferences) { - __ UnpoisonHeapReference(expected); - if (value == expected) { - // Do not unpoison `value`, as it is the same register as - // `expected`, which has just been unpoisoned. - } else { - __ UnpoisonHeapReference(value); - } - } - - __ b(GetExitLabel()); - } - - private: - // The register containing the object holding the marked object reference field. - const Register obj_; - // The offset, index and scale factor to access the reference in `obj_`. - uint32_t offset_; - Location index_; - ScaleFactor scale_factor_; - // Is a null check required? - bool needs_null_check_; - // A temporary register used to hold the lock word of `obj_`; and - // also to hold the original reference value, when the reference is - // marked. - const Register temp1_; - // A temporary register used in the implementation of the CAS, to - // update the object's reference field. - const Register temp2_; - - DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM); -}; - -// Slow path generating a read barrier for a heap reference. -class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { - public: - ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) - : SlowPathCodeARM(instruction), - out_(out), - ref_(ref), - obj_(obj), - offset_(offset), - index_(index) { - DCHECK(kEmitCompilerReadBarrier); - // If `obj` is equal to `out` or `ref`, it means the initial object - // has been overwritten by (or after) the heap object reference load - // to be instrumented, e.g.: - // - // __ LoadFromOffset(kLoadWord, out, out, offset); - // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); - // - // In that case, we have lost the information about the original - // object, and the emitted read barrier cannot work properly. - DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; - DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - LocationSummary* locations = instruction_->GetLocations(); - Register reg_out = out_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); - DCHECK(instruction_->IsInstanceFieldGet() || - instruction_->IsStaticFieldGet() || - instruction_->IsArrayGet() || - instruction_->IsInstanceOf() || - instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) - << "Unexpected instruction in read barrier for heap reference slow path: " - << instruction_->DebugName(); - // The read barrier instrumentation of object ArrayGet - // instructions does not support the HIntermediateAddress - // instruction. - DCHECK(!(instruction_->IsArrayGet() && - instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); - - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - // We may have to change the index's value, but as `index_` is a - // constant member (like other "inputs" of this slow path), - // introduce a copy of it, `index`. - Location index = index_; - if (index_.IsValid()) { - // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. - if (instruction_->IsArrayGet()) { - // Compute the actual memory offset and store it in `index`. - Register index_reg = index_.AsRegister<Register>(); - DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); - if (codegen->IsCoreCalleeSaveRegister(index_reg)) { - // We are about to change the value of `index_reg` (see the - // calls to art::arm::Thumb2Assembler::Lsl and - // art::arm::Thumb2Assembler::AddConstant below), but it has - // not been saved by the previous call to - // art::SlowPathCode::SaveLiveRegisters, as it is a - // callee-save register -- - // art::SlowPathCode::SaveLiveRegisters does not consider - // callee-save registers, as it has been designed with the - // assumption that callee-save registers are supposed to be - // handled by the called function. So, as a callee-save - // register, `index_reg` _would_ eventually be saved onto - // the stack, but it would be too late: we would have - // changed its value earlier. Therefore, we manually save - // it here into another freely available register, - // `free_reg`, chosen of course among the caller-save - // registers (as a callee-save `free_reg` register would - // exhibit the same problem). - // - // Note we could have requested a temporary register from - // the register allocator instead; but we prefer not to, as - // this is a slow path, and we know we can find a - // caller-save register that is available. - Register free_reg = FindAvailableCallerSaveRegister(codegen); - __ Mov(free_reg, index_reg); - index_reg = free_reg; - index = Location::RegisterLocation(index_reg); - } else { - // The initial register stored in `index_` has already been - // saved in the call to art::SlowPathCode::SaveLiveRegisters - // (as it is not a callee-save register), so we can freely - // use it. - } - // Shifting the index value contained in `index_reg` by the scale - // factor (2) cannot overflow in practice, as the runtime is - // unable to allocate object arrays with a size larger than - // 2^26 - 1 (that is, 2^28 - 4 bytes). - __ Lsl(index_reg, index_reg, TIMES_4); - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - __ AddConstant(index_reg, index_reg, offset_); - } else { - // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile - // intrinsics, `index_` is not shifted by a scale factor of 2 - // (as in the case of ArrayGet), as it is actually an offset - // to an object field within an object. - DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); - DCHECK(instruction_->GetLocations()->Intrinsified()); - DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || - (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) - << instruction_->AsInvoke()->GetIntrinsic(); - DCHECK_EQ(offset_, 0U); - DCHECK(index_.IsRegisterPair()); - // UnsafeGet's offset location is a register pair, the low - // part contains the correct offset. - index = index_.ToLow(); - } - } - - // We're moving two or three locations to locations that could - // overlap, so we need a parallel move resolver. - InvokeRuntimeCallingConvention calling_convention; - HParallelMove parallel_move(codegen->GetGraph()->GetArena()); - parallel_move.AddMove(ref_, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimNot, - nullptr); - parallel_move.AddMove(obj_, - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimNot, - nullptr); - if (index.IsValid()) { - parallel_move.AddMove(index, - Location::RegisterLocation(calling_convention.GetRegisterAt(2)), - Primitive::kPrimInt, - nullptr); - codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); - } else { - codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); - __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); - } - arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes< - kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); - arm_codegen->Move32(out_, Location::RegisterLocation(R0)); - - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } - - private: - Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { - size_t ref = static_cast<int>(ref_.AsRegister<Register>()); - size_t obj = static_cast<int>(obj_.AsRegister<Register>()); - for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { - if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { - return static_cast<Register>(i); - } - } - // We shall never fail to find a free caller-save register, as - // there are more than two core caller-save registers on ARM - // (meaning it is possible to find one which is different from - // `ref` and `obj`). - DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); - LOG(FATAL) << "Could not find a free caller-save register"; - UNREACHABLE(); - } - - const Location out_; - const Location ref_; - const Location obj_; - const uint32_t offset_; - // An additional location containing an index to an array. - // Only used for HArrayGet and the UnsafeGetObject & - // UnsafeGetObjectVolatile intrinsics. - const Location index_; - - DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); -}; - -// Slow path generating a read barrier for a GC root. -class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { - public: - ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) - : SlowPathCodeARM(instruction), out_(out), root_(root) { - DCHECK(kEmitCompilerReadBarrier); - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Register reg_out = out_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); - DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) - << "Unexpected instruction in read barrier for GC root slow path: " - << instruction_->DebugName(); - - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); - arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); - arm_codegen->Move32(out_, Location::RegisterLocation(R0)); - - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } - - private: - const Location out_; - const Location root_; - - DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); -}; - -inline Condition ARMCondition(IfCondition cond) { - switch (cond) { - case kCondEQ: return EQ; - case kCondNE: return NE; - case kCondLT: return LT; - case kCondLE: return LE; - case kCondGT: return GT; - case kCondGE: return GE; - case kCondB: return LO; - case kCondBE: return LS; - case kCondA: return HI; - case kCondAE: return HS; - } - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); -} - -// Maps signed condition to unsigned condition. -inline Condition ARMUnsignedCondition(IfCondition cond) { - switch (cond) { - case kCondEQ: return EQ; - case kCondNE: return NE; - // Signed to unsigned. - case kCondLT: return LO; - case kCondLE: return LS; - case kCondGT: return HI; - case kCondGE: return HS; - // Unsigned remain unchanged. - case kCondB: return LO; - case kCondBE: return LS; - case kCondA: return HI; - case kCondAE: return HS; - } - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); -} - -inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { - // The ARM condition codes can express all the necessary branches, see the - // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. - // There is no dex instruction or HIR that would need the missing conditions - // "equal or unordered" or "not equal". - switch (cond) { - case kCondEQ: return EQ; - case kCondNE: return NE /* unordered */; - case kCondLT: return gt_bias ? CC : LT /* unordered */; - case kCondLE: return gt_bias ? LS : LE /* unordered */; - case kCondGT: return gt_bias ? HI /* unordered */ : GT; - case kCondGE: return gt_bias ? CS /* unordered */ : GE; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - -inline Shift ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { - switch (op_kind) { - case HDataProcWithShifterOp::kASR: return ASR; - case HDataProcWithShifterOp::kLSL: return LSL; - case HDataProcWithShifterOp::kLSR: return LSR; - default: - LOG(FATAL) << "Unexpected op kind " << op_kind; - UNREACHABLE(); - } -} - -static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, - Register out, - Register first, - const ShifterOperand& second, - CodeGeneratorARM* codegen) { - if (second.IsImmediate() && second.GetImmediate() == 0) { - const ShifterOperand in = kind == HInstruction::kAnd - ? ShifterOperand(0) - : ShifterOperand(first); - - __ mov(out, in); - } else { - switch (kind) { - case HInstruction::kAdd: - __ add(out, first, second); - break; - case HInstruction::kAnd: - __ and_(out, first, second); - break; - case HInstruction::kOr: - __ orr(out, first, second); - break; - case HInstruction::kSub: - __ sub(out, first, second); - break; - case HInstruction::kXor: - __ eor(out, first, second); - break; - default: - LOG(FATAL) << "Unexpected instruction kind: " << kind; - UNREACHABLE(); - } - } -} - -static void GenerateDataProc(HInstruction::InstructionKind kind, - const Location& out, - const Location& first, - const ShifterOperand& second_lo, - const ShifterOperand& second_hi, - CodeGeneratorARM* codegen) { - const Register first_hi = first.AsRegisterPairHigh<Register>(); - const Register first_lo = first.AsRegisterPairLow<Register>(); - const Register out_hi = out.AsRegisterPairHigh<Register>(); - const Register out_lo = out.AsRegisterPairLow<Register>(); - - if (kind == HInstruction::kAdd) { - __ adds(out_lo, first_lo, second_lo); - __ adc(out_hi, first_hi, second_hi); - } else if (kind == HInstruction::kSub) { - __ subs(out_lo, first_lo, second_lo); - __ sbc(out_hi, first_hi, second_hi); - } else { - GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); - GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); - } -} - -static ShifterOperand GetShifterOperand(Register rm, Shift shift, uint32_t shift_imm) { - return shift_imm == 0 ? ShifterOperand(rm) : ShifterOperand(rm, shift, shift_imm); -} - -static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, CodeGeneratorARM* codegen) { - DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); - DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); - - const LocationSummary* const locations = instruction->GetLocations(); - const uint32_t shift_value = instruction->GetShiftAmount(); - const HInstruction::InstructionKind kind = instruction->GetInstrKind(); - const Location first = locations->InAt(0); - const Location second = locations->InAt(1); - const Location out = locations->Out(); - const Register first_hi = first.AsRegisterPairHigh<Register>(); - const Register first_lo = first.AsRegisterPairLow<Register>(); - const Register out_hi = out.AsRegisterPairHigh<Register>(); - const Register out_lo = out.AsRegisterPairLow<Register>(); - const Register second_hi = second.AsRegisterPairHigh<Register>(); - const Register second_lo = second.AsRegisterPairLow<Register>(); - const Shift shift = ShiftFromOpKind(instruction->GetOpKind()); - - if (shift_value >= 32) { - if (shift == LSL) { - GenerateDataProcInstruction(kind, - out_hi, - first_hi, - ShifterOperand(second_lo, LSL, shift_value - 32), - codegen); - GenerateDataProcInstruction(kind, - out_lo, - first_lo, - ShifterOperand(0), - codegen); - } else if (shift == ASR) { - GenerateDataProc(kind, - out, - first, - GetShifterOperand(second_hi, ASR, shift_value - 32), - ShifterOperand(second_hi, ASR, 31), - codegen); - } else { - DCHECK_EQ(shift, LSR); - GenerateDataProc(kind, - out, - first, - GetShifterOperand(second_hi, LSR, shift_value - 32), - ShifterOperand(0), - codegen); - } - } else { - DCHECK_GT(shift_value, 1U); - DCHECK_LT(shift_value, 32U); - - if (shift == LSL) { - // We are not doing this for HInstruction::kAdd because the output will require - // Location::kOutputOverlap; not applicable to other cases. - if (kind == HInstruction::kOr || kind == HInstruction::kXor) { - GenerateDataProcInstruction(kind, - out_hi, - first_hi, - ShifterOperand(second_hi, LSL, shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_hi, - out_hi, - ShifterOperand(second_lo, LSR, 32 - shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_lo, - first_lo, - ShifterOperand(second_lo, LSL, shift_value), - codegen); - } else { - __ Lsl(IP, second_hi, shift_value); - __ orr(IP, IP, ShifterOperand(second_lo, LSR, 32 - shift_value)); - GenerateDataProc(kind, - out, - first, - ShifterOperand(second_lo, LSL, shift_value), - ShifterOperand(IP), - codegen); - } - } else { - DCHECK(shift == ASR || shift == LSR); - - // We are not doing this for HInstruction::kAdd because the output will require - // Location::kOutputOverlap; not applicable to other cases. - if (kind == HInstruction::kOr || kind == HInstruction::kXor) { - GenerateDataProcInstruction(kind, - out_lo, - first_lo, - ShifterOperand(second_lo, LSR, shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_lo, - out_lo, - ShifterOperand(second_hi, LSL, 32 - shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_hi, - first_hi, - ShifterOperand(second_hi, shift, shift_value), - codegen); - } else { - __ Lsr(IP, second_lo, shift_value); - __ orr(IP, IP, ShifterOperand(second_hi, LSL, 32 - shift_value)); - GenerateDataProc(kind, - out, - first, - ShifterOperand(IP), - ShifterOperand(second_hi, shift, shift_value), - codegen); - } - } - } -} - -static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARM* codegen) { - Primitive::Type type = instruction->InputAt(0)->GetType(); - Location lhs_loc = instruction->GetLocations()->InAt(0); - Location rhs_loc = instruction->GetLocations()->InAt(1); - if (rhs_loc.IsConstant()) { - // 0.0 is the only immediate that can be encoded directly in - // a VCMP instruction. - // - // Both the JLS (section 15.20.1) and the JVMS (section 6.5) - // specify that in a floating-point comparison, positive zero - // and negative zero are considered equal, so we can use the - // literal 0.0 for both cases here. - // - // Note however that some methods (Float.equal, Float.compare, - // Float.compareTo, Double.equal, Double.compare, - // Double.compareTo, Math.max, Math.min, StrictMath.max, - // StrictMath.min) consider 0.0 to be (strictly) greater than - // -0.0. So if we ever translate calls to these methods into a - // HCompare instruction, we must handle the -0.0 case with - // care here. - DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); - if (type == Primitive::kPrimFloat) { - __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); - } else { - DCHECK_EQ(type, Primitive::kPrimDouble); - __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); - } - } else { - if (type == Primitive::kPrimFloat) { - __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); - } else { - DCHECK_EQ(type, Primitive::kPrimDouble); - __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); - } - } -} - -static int64_t AdjustConstantForCondition(int64_t value, - IfCondition* condition, - IfCondition* opposite) { - if (value == 1) { - if (*condition == kCondB) { - value = 0; - *condition = kCondEQ; - *opposite = kCondNE; - } else if (*condition == kCondAE) { - value = 0; - *condition = kCondNE; - *opposite = kCondEQ; - } - } else if (value == -1) { - if (*condition == kCondGT) { - value = 0; - *condition = kCondGE; - *opposite = kCondLT; - } else if (*condition == kCondLE) { - value = 0; - *condition = kCondLT; - *opposite = kCondGE; - } - } - - return value; -} - -static std::pair<Condition, Condition> GenerateLongTestConstant(HCondition* condition, - bool invert, - CodeGeneratorARM* codegen) { - DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = condition->GetLocations(); - IfCondition cond = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - - if (invert) { - std::swap(cond, opposite); - } - - std::pair<Condition, Condition> ret(EQ, NE); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - - DCHECK(right.IsConstant()); - - const Register left_high = left.AsRegisterPairHigh<Register>(); - const Register left_low = left.AsRegisterPairLow<Register>(); - int64_t value = AdjustConstantForCondition(right.GetConstant()->AsLongConstant()->GetValue(), - &cond, - &opposite); - - // Comparisons against 0 are common enough to deserve special attention. - if (value == 0) { - switch (cond) { - case kCondNE: - // x > 0 iff x != 0 when the comparison is unsigned. - case kCondA: - ret = std::make_pair(NE, EQ); - FALLTHROUGH_INTENDED; - case kCondEQ: - // x <= 0 iff x == 0 when the comparison is unsigned. - case kCondBE: - __ orrs(IP, left_low, ShifterOperand(left_high)); - return ret; - case kCondLT: - case kCondGE: - __ cmp(left_high, ShifterOperand(0)); - return std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - // Trivially true or false. - case kCondB: - ret = std::make_pair(NE, EQ); - FALLTHROUGH_INTENDED; - case kCondAE: - __ cmp(left_low, ShifterOperand(left_low)); - return ret; - default: - break; - } - } - - switch (cond) { - case kCondEQ: - case kCondNE: - case kCondB: - case kCondBE: - case kCondA: - case kCondAE: - __ CmpConstant(left_high, High32Bits(value)); - __ it(EQ); - __ cmp(left_low, ShifterOperand(Low32Bits(value)), EQ); - ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); - break; - case kCondLE: - case kCondGT: - // Trivially true or false. - if (value == std::numeric_limits<int64_t>::max()) { - __ cmp(left_low, ShifterOperand(left_low)); - ret = cond == kCondLE ? std::make_pair(EQ, NE) : std::make_pair(NE, EQ); - break; - } - - if (cond == kCondLE) { - DCHECK_EQ(opposite, kCondGT); - cond = kCondLT; - opposite = kCondGE; - } else { - DCHECK_EQ(cond, kCondGT); - DCHECK_EQ(opposite, kCondLE); - cond = kCondGE; - opposite = kCondLT; - } - - value++; - FALLTHROUGH_INTENDED; - case kCondGE: - case kCondLT: - __ CmpConstant(left_low, Low32Bits(value)); - __ sbcs(IP, left_high, ShifterOperand(High32Bits(value))); - ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - break; - default: - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); - } - - return ret; -} - -static std::pair<Condition, Condition> GenerateLongTest(HCondition* condition, - bool invert, - CodeGeneratorARM* codegen) { - DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = condition->GetLocations(); - IfCondition cond = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - - if (invert) { - std::swap(cond, opposite); - } - - std::pair<Condition, Condition> ret; - Location left = locations->InAt(0); - Location right = locations->InAt(1); - - DCHECK(right.IsRegisterPair()); - - switch (cond) { - case kCondEQ: - case kCondNE: - case kCondB: - case kCondBE: - case kCondA: - case kCondAE: - __ cmp(left.AsRegisterPairHigh<Register>(), - ShifterOperand(right.AsRegisterPairHigh<Register>())); - __ it(EQ); - __ cmp(left.AsRegisterPairLow<Register>(), - ShifterOperand(right.AsRegisterPairLow<Register>()), - EQ); - ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); - break; - case kCondLE: - case kCondGT: - if (cond == kCondLE) { - DCHECK_EQ(opposite, kCondGT); - cond = kCondGE; - opposite = kCondLT; - } else { - DCHECK_EQ(cond, kCondGT); - DCHECK_EQ(opposite, kCondLE); - cond = kCondLT; - opposite = kCondGE; - } - - std::swap(left, right); - FALLTHROUGH_INTENDED; - case kCondGE: - case kCondLT: - __ cmp(left.AsRegisterPairLow<Register>(), - ShifterOperand(right.AsRegisterPairLow<Register>())); - __ sbcs(IP, - left.AsRegisterPairHigh<Register>(), - ShifterOperand(right.AsRegisterPairHigh<Register>())); - ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - break; - default: - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); - } - - return ret; -} - -static std::pair<Condition, Condition> GenerateTest(HCondition* condition, - bool invert, - CodeGeneratorARM* codegen) { - const LocationSummary* const locations = condition->GetLocations(); - const Primitive::Type type = condition->GetLeft()->GetType(); - IfCondition cond = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - std::pair<Condition, Condition> ret; - const Location right = locations->InAt(1); - - if (invert) { - std::swap(cond, opposite); - } - - if (type == Primitive::kPrimLong) { - ret = locations->InAt(1).IsConstant() - ? GenerateLongTestConstant(condition, invert, codegen) - : GenerateLongTest(condition, invert, codegen); - } else if (Primitive::IsFloatingPointType(type)) { - GenerateVcmp(condition, codegen); - __ vmstat(); - ret = std::make_pair(ARMFPCondition(cond, condition->IsGtBias()), - ARMFPCondition(opposite, condition->IsGtBias())); - } else { - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - const Register left = locations->InAt(0).AsRegister<Register>(); - - if (right.IsRegister()) { - __ cmp(left, ShifterOperand(right.AsRegister<Register>())); - } else { - DCHECK(right.IsConstant()); - __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); - } - - ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - } - - return ret; -} - -static bool CanGenerateTest(HCondition* condition, ArmAssembler* assembler) { - if (condition->GetLeft()->GetType() == Primitive::kPrimLong) { - const LocationSummary* const locations = condition->GetLocations(); - - if (locations->InAt(1).IsConstant()) { - IfCondition c = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - const int64_t value = AdjustConstantForCondition( - Int64FromConstant(locations->InAt(1).GetConstant()), - &c, - &opposite); - ShifterOperand so; - - if (c < kCondLT || c > kCondGE) { - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the least significant half of the first input to be compared - // is in a low register (the other half is read outside an IT block), and - // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP - // encoding can be used; 0 is always handled, no matter what registers are - // used by the first input. - if (value != 0 && - (!ArmAssembler::IsLowRegister(locations->InAt(0).AsRegisterPairLow<Register>()) || - !IsUint<8>(Low32Bits(value)))) { - return false; - } - } else if (c == kCondLE || c == kCondGT) { - if (value < std::numeric_limits<int64_t>::max() && - !assembler->ShifterOperandCanHold(kNoRegister, - kNoRegister, - SBC, - High32Bits(value + 1), - kCcSet, - &so)) { - return false; - } - } else if (!assembler->ShifterOperandCanHold(kNoRegister, - kNoRegister, - SBC, - High32Bits(value), - kCcSet, - &so)) { - return false; - } - } - } - - return true; -} - -static void GenerateConditionGeneric(HCondition* cond, CodeGeneratorARM* codegen) { - DCHECK(CanGenerateTest(cond, codegen->GetAssembler())); - - const Register out = cond->GetLocations()->Out().AsRegister<Register>(); - const auto condition = GenerateTest(cond, false, codegen); - - __ mov(out, ShifterOperand(0), AL, kCcKeep); - - if (ArmAssembler::IsLowRegister(out)) { - __ it(condition.first); - __ mov(out, ShifterOperand(1), condition.first); - } else { - Label done_label; - Label* const final_label = codegen->GetFinalLabel(cond, &done_label); - - __ b(final_label, condition.second); - __ LoadImmediate(out, 1); - - if (done_label.IsLinked()) { - __ Bind(&done_label); - } - } -} - -static void GenerateEqualLong(HCondition* cond, CodeGeneratorARM* codegen) { - DCHECK_EQ(cond->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = cond->GetLocations(); - IfCondition condition = cond->GetCondition(); - const Register out = locations->Out().AsRegister<Register>(); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - Register left_high = left.AsRegisterPairHigh<Register>(); - Register left_low = left.AsRegisterPairLow<Register>(); - - if (right.IsConstant()) { - IfCondition opposite = cond->GetOppositeCondition(); - const int64_t value = AdjustConstantForCondition(Int64FromConstant(right.GetConstant()), - &condition, - &opposite); - int32_t value_high = -High32Bits(value); - int32_t value_low = -Low32Bits(value); - - // The output uses Location::kNoOutputOverlap. - if (out == left_high) { - std::swap(left_low, left_high); - std::swap(value_low, value_high); - } - - __ AddConstant(out, left_low, value_low); - __ AddConstant(IP, left_high, value_high); - } else { - DCHECK(right.IsRegisterPair()); - __ sub(IP, left_high, ShifterOperand(right.AsRegisterPairHigh<Register>())); - __ sub(out, left_low, ShifterOperand(right.AsRegisterPairLow<Register>())); - } - - // Need to check after calling AdjustConstantForCondition(). - DCHECK(condition == kCondEQ || condition == kCondNE) << condition; - - if (condition == kCondNE && ArmAssembler::IsLowRegister(out)) { - __ orrs(out, out, ShifterOperand(IP)); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - } else { - __ orr(out, out, ShifterOperand(IP)); - codegen->GenerateConditionWithZero(condition, out, out, IP); - } -} - -static void GenerateLongComparesAndJumps(HCondition* cond, - Label* true_label, - Label* false_label, - CodeGeneratorARM* codegen) { - LocationSummary* locations = cond->GetLocations(); - Location left = locations->InAt(0); - Location right = locations->InAt(1); - IfCondition if_cond = cond->GetCondition(); - - Register left_high = left.AsRegisterPairHigh<Register>(); - Register left_low = left.AsRegisterPairLow<Register>(); - IfCondition true_high_cond = if_cond; - IfCondition false_high_cond = cond->GetOppositeCondition(); - Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part - - // Set the conditions for the test, remembering that == needs to be - // decided using the low words. - switch (if_cond) { - case kCondEQ: - case kCondNE: - // Nothing to do. - break; - case kCondLT: - false_high_cond = kCondGT; - break; - case kCondLE: - true_high_cond = kCondLT; - break; - case kCondGT: - false_high_cond = kCondLT; - break; - case kCondGE: - true_high_cond = kCondGT; - break; - case kCondB: - false_high_cond = kCondA; - break; - case kCondBE: - true_high_cond = kCondB; - break; - case kCondA: - false_high_cond = kCondB; - break; - case kCondAE: - true_high_cond = kCondA; - break; - } - if (right.IsConstant()) { - int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); - int32_t val_low = Low32Bits(value); - int32_t val_high = High32Bits(value); - - __ CmpConstant(left_high, val_high); - if (if_cond == kCondNE) { - __ b(true_label, ARMCondition(true_high_cond)); - } else if (if_cond == kCondEQ) { - __ b(false_label, ARMCondition(false_high_cond)); - } else { - __ b(true_label, ARMCondition(true_high_cond)); - __ b(false_label, ARMCondition(false_high_cond)); - } - // Must be equal high, so compare the lows. - __ CmpConstant(left_low, val_low); - } else { - Register right_high = right.AsRegisterPairHigh<Register>(); - Register right_low = right.AsRegisterPairLow<Register>(); - - __ cmp(left_high, ShifterOperand(right_high)); - if (if_cond == kCondNE) { - __ b(true_label, ARMCondition(true_high_cond)); - } else if (if_cond == kCondEQ) { - __ b(false_label, ARMCondition(false_high_cond)); - } else { - __ b(true_label, ARMCondition(true_high_cond)); - __ b(false_label, ARMCondition(false_high_cond)); - } - // Must be equal high, so compare the lows. - __ cmp(left_low, ShifterOperand(right_low)); - } - // The last comparison might be unsigned. - // TODO: optimize cases where this is always true/false - __ b(true_label, final_condition); -} - -static void GenerateConditionLong(HCondition* cond, CodeGeneratorARM* codegen) { - DCHECK_EQ(cond->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = cond->GetLocations(); - IfCondition condition = cond->GetCondition(); - const Register out = locations->Out().AsRegister<Register>(); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - - if (right.IsConstant()) { - IfCondition opposite = cond->GetOppositeCondition(); - - // Comparisons against 0 are common enough to deserve special attention. - if (AdjustConstantForCondition(Int64FromConstant(right.GetConstant()), - &condition, - &opposite) == 0) { - switch (condition) { - case kCondNE: - case kCondA: - if (ArmAssembler::IsLowRegister(out)) { - // We only care if both input registers are 0 or not. - __ orrs(out, - left.AsRegisterPairLow<Register>(), - ShifterOperand(left.AsRegisterPairHigh<Register>())); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - return; - } - - FALLTHROUGH_INTENDED; - case kCondEQ: - case kCondBE: - // We only care if both input registers are 0 or not. - __ orr(out, - left.AsRegisterPairLow<Register>(), - ShifterOperand(left.AsRegisterPairHigh<Register>())); - codegen->GenerateConditionWithZero(condition, out, out); - return; - case kCondLT: - case kCondGE: - // We only care about the sign bit. - FALLTHROUGH_INTENDED; - case kCondAE: - case kCondB: - codegen->GenerateConditionWithZero(condition, out, left.AsRegisterPairHigh<Register>()); - return; - case kCondLE: - case kCondGT: - default: - break; - } - } - } - - if ((condition == kCondEQ || condition == kCondNE) && - // If `out` is a low register, then the GenerateConditionGeneric() - // function generates a shorter code sequence that is still branchless. - (!ArmAssembler::IsLowRegister(out) || !CanGenerateTest(cond, codegen->GetAssembler()))) { - GenerateEqualLong(cond, codegen); - return; - } - - if (CanGenerateTest(cond, codegen->GetAssembler())) { - GenerateConditionGeneric(cond, codegen); - return; - } - - // Convert the jumps into the result. - Label done_label; - Label* const final_label = codegen->GetFinalLabel(cond, &done_label); - Label true_label, false_label; - - GenerateLongComparesAndJumps(cond, &true_label, &false_label, codegen); - - // False case: result = 0. - __ Bind(&false_label); - __ mov(out, ShifterOperand(0)); - __ b(final_label); - - // True case: result = 1. - __ Bind(&true_label); - __ mov(out, ShifterOperand(1)); - - if (done_label.IsLinked()) { - __ Bind(&done_label); - } -} - -static void GenerateConditionIntegralOrNonPrimitive(HCondition* cond, CodeGeneratorARM* codegen) { - const Primitive::Type type = cond->GetLeft()->GetType(); - - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - if (type == Primitive::kPrimLong) { - GenerateConditionLong(cond, codegen); - return; - } - - const LocationSummary* const locations = cond->GetLocations(); - IfCondition condition = cond->GetCondition(); - Register in = locations->InAt(0).AsRegister<Register>(); - const Register out = locations->Out().AsRegister<Register>(); - const Location right = cond->GetLocations()->InAt(1); - int64_t value; - - if (right.IsConstant()) { - IfCondition opposite = cond->GetOppositeCondition(); - - value = AdjustConstantForCondition(Int64FromConstant(right.GetConstant()), - &condition, - &opposite); - - // Comparisons against 0 are common enough to deserve special attention. - if (value == 0) { - switch (condition) { - case kCondNE: - case kCondA: - if (ArmAssembler::IsLowRegister(out) && out == in) { - __ cmp(out, ShifterOperand(0)); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - return; - } - - FALLTHROUGH_INTENDED; - case kCondEQ: - case kCondBE: - case kCondLT: - case kCondGE: - case kCondAE: - case kCondB: - codegen->GenerateConditionWithZero(condition, out, in); - return; - case kCondLE: - case kCondGT: - default: - break; - } - } - } - - if (condition == kCondEQ || condition == kCondNE) { - ShifterOperand operand; - - if (right.IsConstant()) { - operand = ShifterOperand(value); - } else if (out == right.AsRegister<Register>()) { - // Avoid 32-bit instructions if possible. - operand = ShifterOperand(in); - in = right.AsRegister<Register>(); - } else { - operand = ShifterOperand(right.AsRegister<Register>()); - } - - if (condition == kCondNE && ArmAssembler::IsLowRegister(out)) { - __ subs(out, in, operand); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - } else { - __ sub(out, in, operand); - codegen->GenerateConditionWithZero(condition, out, out); - } - - return; - } - - GenerateConditionGeneric(cond, codegen); -} - -static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) { - const Primitive::Type type = constant->GetType(); - bool ret = false; - - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - if (type == Primitive::kPrimLong) { - const uint64_t value = constant->AsLongConstant()->GetValueAsUint64(); - - ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value)); - } else { - ret = IsUint<8>(CodeGenerator::GetInt32ValueOf(constant)); - } - - return ret; -} - -static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) { - DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); - - if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) { - return Location::ConstantLocation(constant->AsConstant()); - } - - return Location::RequiresRegister(); -} - -static bool CanGenerateConditionalMove(const Location& out, const Location& src) { - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that we are not dealing with floating-point output (there is no - // 16-bit VMOV encoding). - if (!out.IsRegister() && !out.IsRegisterPair()) { - return false; - } - - // For constants, we also check that the output is in one or two low registers, - // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit - // MOV encoding can be used. - if (src.IsConstant()) { - if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) { - return false; - } - - if (out.IsRegister()) { - if (!ArmAssembler::IsLowRegister(out.AsRegister<Register>())) { - return false; - } - } else { - DCHECK(out.IsRegisterPair()); - - if (!ArmAssembler::IsLowRegister(out.AsRegisterPairHigh<Register>())) { - return false; - } - } - } - - return true; -} - -#undef __ -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT - -Label* CodeGeneratorARM::GetFinalLabel(HInstruction* instruction, Label* final_label) { - DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck()); - DCHECK(!instruction->IsInvoke() || !instruction->GetLocations()->CanCall()); - - const HBasicBlock* const block = instruction->GetBlock(); - const HLoopInformation* const info = block->GetLoopInformation(); - HInstruction* const next = instruction->GetNext(); - - // Avoid a branch to a branch. - if (next->IsGoto() && (info == nullptr || - !info->IsBackEdge(*block) || - !info->HasSuspendCheck())) { - final_label = GetLabelOf(next->AsGoto()->GetSuccessor()); - } - - return final_label; -} - -void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { - stream << Register(reg); -} - -void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { - stream << SRegister(reg); -} - -size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { - __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); - return kArmWordSize; -} - -size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { - __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); - return kArmWordSize; -} - -size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); - return kArmWordSize; -} - -size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); - return kArmWordSize; -} - -CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, - const ArmInstructionSetFeatures& isa_features, - const CompilerOptions& compiler_options, - OptimizingCompilerStats* stats) - : CodeGenerator(graph, - kNumberOfCoreRegisters, - kNumberOfSRegisters, - kNumberOfRegisterPairs, - ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), - arraysize(kCoreCalleeSaves)), - ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), - arraysize(kFpuCalleeSaves)), - compiler_options, - stats), - block_labels_(nullptr), - location_builder_(graph, this), - instruction_visitor_(graph, this), - move_resolver_(graph->GetArena(), this), - assembler_(graph->GetArena()), - isa_features_(isa_features), - uint32_literals_(std::less<uint32_t>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - baker_read_barrier_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - jit_string_patches_(StringReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - jit_class_patches_(TypeReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { - // Always save the LR register to mimic Quick. - AddAllocatedRegister(Location::RegisterLocation(LR)); -} - -void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { - // Ensure that we fix up branches and literal loads and emit the literal pool. - __ FinalizeCode(); - - // Adjust native pc offsets in stack maps. - for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { - uint32_t old_position = - stack_map_stream_.GetStackMap(i).native_pc_code_offset.Uint32Value(kThumb2); - uint32_t new_position = __ GetAdjustedPosition(old_position); - stack_map_stream_.SetStackMapNativePcOffset(i, new_position); - } - // Adjust pc offsets for the disassembly information. - if (disasm_info_ != nullptr) { - GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); - frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); - frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); - for (auto& it : *disasm_info_->GetInstructionIntervals()) { - it.second.start = __ GetAdjustedPosition(it.second.start); - it.second.end = __ GetAdjustedPosition(it.second.end); - } - for (auto& it : *disasm_info_->GetSlowPathIntervals()) { - it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); - it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); - } - } - - CodeGenerator::Finalize(allocator); -} - -void CodeGeneratorARM::SetupBlockedRegisters() const { - // Stack register, LR and PC are always reserved. - blocked_core_registers_[SP] = true; - blocked_core_registers_[LR] = true; - blocked_core_registers_[PC] = true; - - // Reserve thread register. - blocked_core_registers_[TR] = true; - - // Reserve temp register. - blocked_core_registers_[IP] = true; - - if (GetGraph()->IsDebuggable()) { - // Stubs do not save callee-save floating point registers. If the graph - // is debuggable, we need to deal with these registers differently. For - // now, just block them. - for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { - blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; - } - } -} - -InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) - : InstructionCodeGenerator(graph, codegen), - assembler_(codegen->GetAssembler()), - codegen_(codegen) {} - -void CodeGeneratorARM::ComputeSpillMask() { - core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; - DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; - // There is no easy instruction to restore just the PC on thumb2. We spill and - // restore another arbitrary register. - core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); - fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; - // We use vpush and vpop for saving and restoring floating point registers, which take - // a SRegister and the number of registers to save/restore after that SRegister. We - // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, - // but in the range. - if (fpu_spill_mask_ != 0) { - uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); - uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); - for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { - fpu_spill_mask_ |= (1 << i); - } - } -} - -static dwarf::Reg DWARFReg(Register reg) { - return dwarf::Reg::ArmCore(static_cast<int>(reg)); -} - -static dwarf::Reg DWARFReg(SRegister reg) { - return dwarf::Reg::ArmFp(static_cast<int>(reg)); -} - -void CodeGeneratorARM::GenerateFrameEntry() { - bool skip_overflow_check = - IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); - DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); - __ Bind(&frame_entry_label_); - - if (HasEmptyFrame()) { - return; - } - - if (!skip_overflow_check) { - __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); - __ LoadFromOffset(kLoadWord, IP, IP, 0); - RecordPcInfo(nullptr, 0); - } - - __ PushList(core_spill_mask_); - __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); - __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); - if (fpu_spill_mask_ != 0) { - SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); - __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); - __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); - __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); - } - - if (GetGraph()->HasShouldDeoptimizeFlag()) { - // Initialize should_deoptimize flag to 0. - __ mov(IP, ShifterOperand(0)); - __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); - } - - int adjust = GetFrameSize() - FrameEntrySpillSize(); - __ AddConstant(SP, -adjust); - __ cfi().AdjustCFAOffset(adjust); - - // Save the current method if we need it. Note that we do not - // do this in HCurrentMethod, as the instruction might have been removed - // in the SSA graph. - if (RequiresCurrentMethod()) { - __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); - } -} - -void CodeGeneratorARM::GenerateFrameExit() { - if (HasEmptyFrame()) { - __ bx(LR); - return; - } - __ cfi().RememberState(); - int adjust = GetFrameSize() - FrameEntrySpillSize(); - __ AddConstant(SP, adjust); - __ cfi().AdjustCFAOffset(-adjust); - if (fpu_spill_mask_ != 0) { - SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); - __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); - __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); - __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); - } - // Pop LR into PC to return. - DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); - uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; - __ PopList(pop_mask); - __ cfi().RestoreState(); - __ cfi().DefCFAOffset(GetFrameSize()); -} - -void CodeGeneratorARM::Bind(HBasicBlock* block) { - Label* label = GetLabelOf(block); - __ BindTrackedLabel(label); -} - -Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - uint32_t index = gp_index_++; - uint32_t stack_index = stack_index_++; - if (index < calling_convention.GetNumberOfRegisters()) { - return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); - } else { - return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimLong: { - uint32_t index = gp_index_; - uint32_t stack_index = stack_index_; - gp_index_ += 2; - stack_index_ += 2; - if (index + 1 < calling_convention.GetNumberOfRegisters()) { - if (calling_convention.GetRegisterAt(index) == R1) { - // Skip R1, and use R2_R3 instead. - gp_index_++; - index++; - } - } - if (index + 1 < calling_convention.GetNumberOfRegisters()) { - DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, - calling_convention.GetRegisterAt(index + 1)); - - return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), - calling_convention.GetRegisterAt(index + 1)); - } else { - return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimFloat: { - uint32_t stack_index = stack_index_++; - if (float_index_ % 2 == 0) { - float_index_ = std::max(double_index_, float_index_); - } - if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { - return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); - } else { - return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimDouble: { - double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); - uint32_t stack_index = stack_index_; - stack_index_ += 2; - if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { - uint32_t index = double_index_; - double_index_ += 2; - Location result = Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(index), - calling_convention.GetFpuRegisterAt(index + 1)); - DCHECK(ExpectedPairLayout(result)); - return result; - } else { - return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unexpected parameter type " << type; - break; - } - return Location::NoLocation(); -} - -Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - return Location::RegisterLocation(R0); - } - - case Primitive::kPrimFloat: { - return Location::FpuRegisterLocation(S0); - } - - case Primitive::kPrimLong: { - return Location::RegisterPairLocation(R0, R1); - } - - case Primitive::kPrimDouble: { - return Location::FpuRegisterPairLocation(S0, S1); - } - - case Primitive::kPrimVoid: - return Location::NoLocation(); - } - - UNREACHABLE(); -} - -Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { - return Location::RegisterLocation(kMethodRegisterArgument); -} - -void CodeGeneratorARM::Move32(Location destination, Location source) { - if (source.Equals(destination)) { - return; - } - if (destination.IsRegister()) { - if (source.IsRegister()) { - __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); - } else if (source.IsFpuRegister()) { - __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); - } else { - __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); - } - } else if (destination.IsFpuRegister()) { - if (source.IsRegister()) { - __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); - } else if (source.IsFpuRegister()) { - __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); - } else { - __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); - } - } else { - DCHECK(destination.IsStackSlot()) << destination; - if (source.IsRegister()) { - __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); - } else if (source.IsFpuRegister()) { - __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); - } else { - DCHECK(source.IsStackSlot()) << source; - __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } -} - -void CodeGeneratorARM::Move64(Location destination, Location source) { - if (source.Equals(destination)) { - return; - } - if (destination.IsRegisterPair()) { - if (source.IsRegisterPair()) { - EmitParallelMoves( - Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), - Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), - Primitive::kPrimInt, - Location::RegisterLocation(source.AsRegisterPairLow<Register>()), - Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), - Primitive::kPrimInt); - } else if (source.IsFpuRegister()) { - UNIMPLEMENTED(FATAL); - } else if (source.IsFpuRegisterPair()) { - __ vmovrrd(destination.AsRegisterPairLow<Register>(), - destination.AsRegisterPairHigh<Register>(), - FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); - } else { - DCHECK(source.IsDoubleStackSlot()); - DCHECK(ExpectedPairLayout(destination)); - __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), - SP, source.GetStackIndex()); - } - } else if (destination.IsFpuRegisterPair()) { - if (source.IsDoubleStackSlot()) { - __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - SP, - source.GetStackIndex()); - } else if (source.IsRegisterPair()) { - __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - source.AsRegisterPairLow<Register>(), - source.AsRegisterPairHigh<Register>()); - } else { - UNIMPLEMENTED(FATAL); - } - } else { - DCHECK(destination.IsDoubleStackSlot()); - if (source.IsRegisterPair()) { - // No conflict possible, so just do the moves. - if (source.AsRegisterPairLow<Register>() == R1) { - DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); - __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); - __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); - } else { - __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), - SP, destination.GetStackIndex()); - } - } else if (source.IsFpuRegisterPair()) { - __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), - SP, - destination.GetStackIndex()); - } else { - DCHECK(source.IsDoubleStackSlot()); - EmitParallelMoves( - Location::StackSlot(source.GetStackIndex()), - Location::StackSlot(destination.GetStackIndex()), - Primitive::kPrimInt, - Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), - Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), - Primitive::kPrimInt); - } - } -} - -void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { - DCHECK(location.IsRegister()); - __ LoadImmediate(location.AsRegister<Register>(), value); -} - -void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { - HParallelMove move(GetGraph()->GetArena()); - move.AddMove(src, dst, dst_type, nullptr); - GetMoveResolver()->EmitNativeCode(&move); -} - -void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { - if (location.IsRegister()) { - locations->AddTemp(location); - } else if (location.IsRegisterPair()) { - locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); - locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); - } else { - UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; - } -} - -void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, - HInstruction* instruction, - uint32_t dex_pc, - SlowPathCode* slow_path) { - ValidateInvokeRuntime(entrypoint, instruction, slow_path); - GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); - if (EntrypointRequiresStackMap(entrypoint)) { - RecordPcInfo(instruction, dex_pc, slow_path); - } -} - -void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, - HInstruction* instruction, - SlowPathCode* slow_path) { - ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); - GenerateInvokeRuntime(entry_point_offset); -} - -void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { - __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); - __ blx(LR); -} - -void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { - DCHECK(!successor->IsExitBlock()); - - HBasicBlock* block = got->GetBlock(); - HInstruction* previous = got->GetPrevious(); - - HLoopInformation* info = block->GetLoopInformation(); - if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { - codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); - GenerateSuspendCheck(info->GetSuspendCheck(), successor); - return; - } - - if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { - GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); - } - if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { - __ b(codegen_->GetLabelOf(successor)); - } -} - -void LocationsBuilderARM::VisitGoto(HGoto* got) { - got->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { - HandleGoto(got, got->GetSuccessor()); -} - -void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { - try_boundary->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { - HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); - if (!successor->IsExitBlock()) { - HandleGoto(try_boundary, successor); - } -} - -void LocationsBuilderARM::VisitExit(HExit* exit) { - exit->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { -} - -void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, - Label* true_target_in, - Label* false_target_in) { - if (CanGenerateTest(condition, codegen_->GetAssembler())) { - Label* non_fallthrough_target; - bool invert; - bool emit_both_branches; - - if (true_target_in == nullptr) { - // The true target is fallthrough. - DCHECK(false_target_in != nullptr); - non_fallthrough_target = false_target_in; - invert = true; - emit_both_branches = false; - } else { - // Either the false target is fallthrough, or there is no fallthrough - // and both branches must be emitted. - non_fallthrough_target = true_target_in; - invert = false; - emit_both_branches = (false_target_in != nullptr); - } - - const auto cond = GenerateTest(condition, invert, codegen_); - - __ b(non_fallthrough_target, cond.first); - - if (emit_both_branches) { - // No target falls through, we need to branch. - __ b(false_target_in); - } - - return; - } - - // Generated branching requires both targets to be explicit. If either of the - // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. - Label fallthrough_target; - Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; - Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; - - DCHECK_EQ(condition->InputAt(0)->GetType(), Primitive::kPrimLong); - GenerateLongComparesAndJumps(condition, true_target, false_target, codegen_); - - if (false_target != &fallthrough_target) { - __ b(false_target); - } - - if (fallthrough_target.IsLinked()) { - __ Bind(&fallthrough_target); - } -} - -void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, - size_t condition_input_index, - Label* true_target, - Label* false_target) { - HInstruction* cond = instruction->InputAt(condition_input_index); - - if (true_target == nullptr && false_target == nullptr) { - // Nothing to do. The code always falls through. - return; - } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against "true" (integer value 1). - if (cond->AsIntConstant()->IsTrue()) { - if (true_target != nullptr) { - __ b(true_target); - } - } else { - DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); - if (false_target != nullptr) { - __ b(false_target); - } - } - return; - } - - // The following code generates these patterns: - // (1) true_target == nullptr && false_target != nullptr - // - opposite condition true => branch to false_target - // (2) true_target != nullptr && false_target == nullptr - // - condition true => branch to true_target - // (3) true_target != nullptr && false_target != nullptr - // - condition true => branch to true_target - // - branch to false_target - if (IsBooleanValueOrMaterializedCondition(cond)) { - // Condition has been materialized, compare the output to 0. - Location cond_val = instruction->GetLocations()->InAt(condition_input_index); - DCHECK(cond_val.IsRegister()); - if (true_target == nullptr) { - __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); - } else { - __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); - } - } else { - // Condition has not been materialized. Use its inputs as the comparison and - // its condition as the branch condition. - HCondition* condition = cond->AsCondition(); - - // If this is a long or FP comparison that has been folded into - // the HCondition, generate the comparison directly. - Primitive::Type type = condition->InputAt(0)->GetType(); - if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { - GenerateCompareTestAndBranch(condition, true_target, false_target); - return; - } - - Label* non_fallthrough_target; - Condition arm_cond; - LocationSummary* locations = cond->GetLocations(); - DCHECK(locations->InAt(0).IsRegister()); - Register left = locations->InAt(0).AsRegister<Register>(); - Location right = locations->InAt(1); - - if (true_target == nullptr) { - arm_cond = ARMCondition(condition->GetOppositeCondition()); - non_fallthrough_target = false_target; - } else { - arm_cond = ARMCondition(condition->GetCondition()); - non_fallthrough_target = true_target; - } - - if (right.IsConstant() && (arm_cond == NE || arm_cond == EQ) && - CodeGenerator::GetInt32ValueOf(right.GetConstant()) == 0) { - if (arm_cond == EQ) { - __ CompareAndBranchIfZero(left, non_fallthrough_target); - } else { - DCHECK_EQ(arm_cond, NE); - __ CompareAndBranchIfNonZero(left, non_fallthrough_target); - } - } else { - if (right.IsRegister()) { - __ cmp(left, ShifterOperand(right.AsRegister<Register>())); - } else { - DCHECK(right.IsConstant()); - __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); - } - - __ b(non_fallthrough_target, arm_cond); - } - } - - // If neither branch falls through (case 3), the conditional branch to `true_target` - // was already emitted (case 2) and we need to emit a jump to `false_target`. - if (true_target != nullptr && false_target != nullptr) { - __ b(false_target); - } -} - -void LocationsBuilderARM::VisitIf(HIf* if_instr) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); - if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { - locations->SetInAt(0, Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { - HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); - HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); - Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? - nullptr : codegen_->GetLabelOf(true_successor); - Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? - nullptr : codegen_->GetLabelOf(false_successor); - GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); -} - -void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { - LocationSummary* locations = new (GetGraph()->GetArena()) - LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); - InvokeRuntimeCallingConvention calling_convention; - RegisterSet caller_saves = RegisterSet::Empty(); - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetCustomSlowPathCallerSaves(caller_saves); - if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { - locations->SetInAt(0, Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { - SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); - GenerateTestAndBranch(deoptimize, - /* condition_input_index */ 0, - slow_path->GetEntryLabel(), - /* false_target */ nullptr); -} - -void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { - LocationSummary* locations = new (GetGraph()->GetArena()) - LocationSummary(flag, LocationSummary::kNoCall); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { - __ LoadFromOffset(kLoadWord, - flag->GetLocations()->Out().AsRegister<Register>(), - SP, - codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); -} - -void LocationsBuilderARM::VisitSelect(HSelect* select) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); - const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType()); - - if (is_floating_point) { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue())); - } else { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue())); - } - - if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { - locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition())); - // The code generator handles overlap with the values, but not with the condition. - locations->SetOut(Location::SameAsFirstInput()); - } else if (is_floating_point) { - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - } else { - if (!locations->InAt(1).IsConstant()) { - locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue())); - } - - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } -} - -void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { - HInstruction* const condition = select->GetCondition(); - const LocationSummary* const locations = select->GetLocations(); - const Primitive::Type type = select->GetType(); - const Location first = locations->InAt(0); - const Location out = locations->Out(); - const Location second = locations->InAt(1); - Location src; - - if (condition->IsIntConstant()) { - if (condition->AsIntConstant()->IsFalse()) { - src = first; - } else { - src = second; - } - - codegen_->MoveLocation(out, src, type); - return; - } - - if (!Primitive::IsFloatingPointType(type) && - (IsBooleanValueOrMaterializedCondition(condition) || - CanGenerateTest(condition->AsCondition(), codegen_->GetAssembler()))) { - bool invert = false; - - if (out.Equals(second)) { - src = first; - invert = true; - } else if (out.Equals(first)) { - src = second; - } else if (second.IsConstant()) { - DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant())); - src = second; - } else if (first.IsConstant()) { - DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant())); - src = first; - invert = true; - } else { - src = second; - } - - if (CanGenerateConditionalMove(out, src)) { - if (!out.Equals(first) && !out.Equals(second)) { - codegen_->MoveLocation(out, src.Equals(first) ? second : first, type); - } - - std::pair<Condition, Condition> cond; - - if (IsBooleanValueOrMaterializedCondition(condition)) { - __ CmpConstant(locations->InAt(2).AsRegister<Register>(), 0); - cond = invert ? std::make_pair(EQ, NE) : std::make_pair(NE, EQ); - } else { - cond = GenerateTest(condition->AsCondition(), invert, codegen_); - } - - if (out.IsRegister()) { - ShifterOperand operand; - - if (src.IsConstant()) { - operand = ShifterOperand(CodeGenerator::GetInt32ValueOf(src.GetConstant())); - } else { - DCHECK(src.IsRegister()); - operand = ShifterOperand(src.AsRegister<Register>()); - } - - __ it(cond.first); - __ mov(out.AsRegister<Register>(), operand, cond.first); - } else { - DCHECK(out.IsRegisterPair()); - - ShifterOperand operand_high; - ShifterOperand operand_low; - - if (src.IsConstant()) { - const int64_t value = src.GetConstant()->AsLongConstant()->GetValue(); - - operand_high = ShifterOperand(High32Bits(value)); - operand_low = ShifterOperand(Low32Bits(value)); - } else { - DCHECK(src.IsRegisterPair()); - operand_high = ShifterOperand(src.AsRegisterPairHigh<Register>()); - operand_low = ShifterOperand(src.AsRegisterPairLow<Register>()); - } - - __ it(cond.first); - __ mov(out.AsRegisterPairLow<Register>(), operand_low, cond.first); - __ it(cond.first); - __ mov(out.AsRegisterPairHigh<Register>(), operand_high, cond.first); - } - - return; - } - } - - Label* false_target = nullptr; - Label* true_target = nullptr; - Label select_end; - Label* target = codegen_->GetFinalLabel(select, &select_end); - - if (out.Equals(second)) { - true_target = target; - src = first; - } else { - false_target = target; - src = second; - - if (!out.Equals(first)) { - codegen_->MoveLocation(out, first, type); - } - } - - GenerateTestAndBranch(select, 2, true_target, false_target); - codegen_->MoveLocation(out, src, type); - - if (select_end.IsLinked()) { - __ Bind(&select_end); - } -} - -void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { - new (GetGraph()->GetArena()) LocationSummary(info); -} - -void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { - // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. -} - -void CodeGeneratorARM::GenerateNop() { - __ nop(); -} - -// `temp` is an extra temporary register that is used for some conditions; -// callers may not specify it, in which case the method will use a scratch -// register instead. -void CodeGeneratorARM::GenerateConditionWithZero(IfCondition condition, - Register out, - Register in, - Register temp) { - switch (condition) { - case kCondEQ: - // x <= 0 iff x == 0 when the comparison is unsigned. - case kCondBE: - if (temp == kNoRegister || (ArmAssembler::IsLowRegister(out) && out != in)) { - temp = out; - } - - // Avoid 32-bit instructions if possible; note that `in` and `temp` must be - // different as well. - if (ArmAssembler::IsLowRegister(in) && ArmAssembler::IsLowRegister(temp) && in != temp) { - // temp = - in; only 0 sets the carry flag. - __ rsbs(temp, in, ShifterOperand(0)); - - if (out == in) { - std::swap(in, temp); - } - - // out = - in + in + carry = carry - __ adc(out, temp, ShifterOperand(in)); - } else { - // If `in` is 0, then it has 32 leading zeros, and less than that otherwise. - __ clz(out, in); - // Any number less than 32 logically shifted right by 5 bits results in 0; - // the same operation on 32 yields 1. - __ Lsr(out, out, 5); - } - - break; - case kCondNE: - // x > 0 iff x != 0 when the comparison is unsigned. - case kCondA: - if (out == in) { - if (temp == kNoRegister || in == temp) { - temp = IP; - } - } else if (temp == kNoRegister || !ArmAssembler::IsLowRegister(temp)) { - temp = out; - } - - // temp = in - 1; only 0 does not set the carry flag. - __ subs(temp, in, ShifterOperand(1)); - // out = in + ~temp + carry = in + (-(in - 1) - 1) + carry = in - in + 1 - 1 + carry = carry - __ sbc(out, in, ShifterOperand(temp)); - break; - case kCondGE: - __ mvn(out, ShifterOperand(in)); - in = out; - FALLTHROUGH_INTENDED; - case kCondLT: - // We only care about the sign bit. - __ Lsr(out, in, 31); - break; - case kCondAE: - // Trivially true. - __ mov(out, ShifterOperand(1)); - break; - case kCondB: - // Trivially false. - __ mov(out, ShifterOperand(0)); - break; - default: - LOG(FATAL) << "Unexpected condition " << condition; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::HandleCondition(HCondition* cond) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); - // Handle the long/FP comparisons made in instruction simplification. - switch (cond->InputAt(0)->GetType()) { - case Primitive::kPrimLong: - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); - if (!cond->IsEmittedAtUseSite()) { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } - break; - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); - if (!cond->IsEmittedAtUseSite()) { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } - break; - - default: - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); - if (!cond->IsEmittedAtUseSite()) { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } - } -} - -void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { - if (cond->IsEmittedAtUseSite()) { - return; - } - - const Primitive::Type type = cond->GetLeft()->GetType(); - - if (Primitive::IsFloatingPointType(type)) { - GenerateConditionGeneric(cond, codegen_); - return; - } - - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - const IfCondition condition = cond->GetCondition(); - - // A condition with only one boolean input, or two boolean inputs without being equality or - // inequality results from transformations done by the instruction simplifier, and is handled - // as a regular condition with integral inputs. - if (type == Primitive::kPrimBoolean && - cond->GetRight()->GetType() == Primitive::kPrimBoolean && - (condition == kCondEQ || condition == kCondNE)) { - const LocationSummary* const locations = cond->GetLocations(); - Register left = locations->InAt(0).AsRegister<Register>(); - const Register out = locations->Out().AsRegister<Register>(); - const Location right_loc = locations->InAt(1); - - // The constant case is handled by the instruction simplifier. - DCHECK(!right_loc.IsConstant()); - - Register right = right_loc.AsRegister<Register>(); - - // Avoid 32-bit instructions if possible. - if (out == right) { - std::swap(left, right); - } - - __ eor(out, left, ShifterOperand(right)); - - if (condition == kCondEQ) { - __ eor(out, out, ShifterOperand(1)); - } - - return; - } - - GenerateConditionIntegralOrNonPrimitive(cond, codegen_); -} - -void LocationsBuilderARM::VisitEqual(HEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitBelow(HBelow* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitAbove(HAbove* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitConstructorFence(HConstructorFence* constructor_fence) { - constructor_fence->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitConstructorFence( - HConstructorFence* constructor_fence ATTRIBUTE_UNUSED) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore); -} - -void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { - memory_barrier->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { - codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); -} - -void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { - ret->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { - codegen_->GenerateFrameExit(); -} - -void LocationsBuilderARM::VisitReturn(HReturn* ret) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); - locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); -} - -void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { - codegen_->GenerateFrameExit(); -} - -void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { - // The trampoline uses the same calling convention as dex calling conventions, - // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain - // the method_idx. - HandleInvoke(invoke); -} - -void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { - codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); -} - -void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { - // Explicit clinit checks triggered by static invokes must have been pruned by - // art::PrepareForRegisterAllocation. - DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); - - IntrinsicLocationsBuilderARM intrinsic(codegen_); - if (intrinsic.TryDispatch(invoke)) { - if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); - } - return; - } - - HandleInvoke(invoke); - - // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. - if (invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); - } -} - -static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { - if (invoke->GetLocations()->Intrinsified()) { - IntrinsicCodeGeneratorARM intrinsic(codegen); - intrinsic.Dispatch(invoke); - return true; - } - return false; -} - -void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { - // Explicit clinit checks triggered by static invokes must have been pruned by - // art::PrepareForRegisterAllocation. - DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); - - if (TryGenerateIntrinsicCode(invoke, codegen_)) { - return; - } - - LocationSummary* locations = invoke->GetLocations(); - codegen_->GenerateStaticOrDirectCall( - invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); -} - -void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { - InvokeDexCallingConventionVisitorARM calling_convention_visitor; - CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); -} - -void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { - IntrinsicLocationsBuilderARM intrinsic(codegen_); - if (intrinsic.TryDispatch(invoke)) { - return; - } - - HandleInvoke(invoke); -} - -void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { - if (TryGenerateIntrinsicCode(invoke, codegen_)) { - return; - } - - codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); - DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); -} - -void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { - HandleInvoke(invoke); - // Add the hidden argument. - invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); -} - -void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { - // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. - LocationSummary* locations = invoke->GetLocations(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); - Location receiver = locations->InAt(0); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - - // Set the hidden argument. This is safe to do this here, as R12 - // won't be modified thereafter, before the `blx` (call) instruction. - DCHECK_EQ(R12, hidden_reg); - __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); - - if (receiver.IsStackSlot()) { - __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); - // /* HeapReference<Class> */ temp = temp->klass_ - __ LoadFromOffset(kLoadWord, temp, temp, class_offset); - } else { - // /* HeapReference<Class> */ temp = receiver->klass_ - __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); - } - codegen_->MaybeRecordImplicitNullCheck(invoke); - // Instead of simply (possibly) unpoisoning `temp` here, we should - // emit a read barrier for the previous class reference load. - // However this is not required in practice, as this is an - // intermediate/temporary reference and because the current - // concurrent copying collector keeps the from-space memory - // intact/accessible until the end of the marking phase (the - // concurrent copying collector may not in the future). - __ MaybeUnpoisonHeapReference(temp); - __ LoadFromOffset(kLoadWord, temp, temp, - mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); - uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( - invoke->GetImtIndex(), kArmPointerSize)); - // temp = temp->GetImtEntryAt(method_offset); - __ LoadFromOffset(kLoadWord, temp, temp, method_offset); - uint32_t entry_point = - ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); - // LR = temp->GetEntryPoint(); - __ LoadFromOffset(kLoadWord, LR, temp, entry_point); - // LR(); - __ blx(LR); - DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); -} - -void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { - HandleInvoke(invoke); -} - -void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { - codegen_->GenerateInvokePolymorphicCall(invoke); -} - -void LocationsBuilderARM::VisitNeg(HNeg* neg) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); - switch (neg->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - break; - } - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { - LocationSummary* locations = neg->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - switch (neg->GetResultType()) { - case Primitive::kPrimInt: - DCHECK(in.IsRegister()); - __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); - break; - - case Primitive::kPrimLong: - DCHECK(in.IsRegisterPair()); - // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) - __ rsbs(out.AsRegisterPairLow<Register>(), - in.AsRegisterPairLow<Register>(), - ShifterOperand(0)); - // We cannot emit an RSC (Reverse Subtract with Carry) - // instruction here, as it does not exist in the Thumb-2 - // instruction set. We use the following approach - // using SBC and SUB instead. - // - // out.hi = -C - __ sbc(out.AsRegisterPairHigh<Register>(), - out.AsRegisterPairHigh<Register>(), - ShifterOperand(out.AsRegisterPairHigh<Register>())); - // out.hi = out.hi - in.hi - __ sub(out.AsRegisterPairHigh<Register>(), - out.AsRegisterPairHigh<Register>(), - ShifterOperand(in.AsRegisterPairHigh<Register>())); - break; - - case Primitive::kPrimFloat: - DCHECK(in.IsFpuRegister()); - __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); - break; - - case Primitive::kPrimDouble: - DCHECK(in.IsFpuRegisterPair()); - __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - break; - - default: - LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); - } -} - -void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { - Primitive::Type result_type = conversion->GetResultType(); - Primitive::Type input_type = conversion->GetInputType(); - DCHECK_NE(result_type, input_type); - - // The float-to-long, double-to-long and long-to-float type conversions - // rely on a call to the runtime. - LocationSummary::CallKind call_kind = - (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) - && result_type == Primitive::kPrimLong) - || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) - ? LocationSummary::kCallOnMainOnly - : LocationSummary::kNoCall; - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); - - // The Java language does not allow treating boolean as an integral type but - // our bit representation makes it safe. - - switch (result_type) { - case Primitive::kPrimByte: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to byte is a result of code transformations. - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-byte' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimShort: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to short is a result of code transformations. - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-short' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimInt: - switch (input_type) { - case Primitive::kPrimLong: - // Processing a Dex `long-to-int' instruction. - locations->SetInAt(0, Location::Any()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-int' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-int' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimLong: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-long' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - case Primitive::kPrimFloat: { - // Processing a Dex `float-to-long' instruction. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterLocation( - calling_convention.GetFpuRegisterAt(0))); - locations->SetOut(Location::RegisterPairLocation(R0, R1)); - break; - } - - case Primitive::kPrimDouble: { - // Processing a Dex `double-to-long' instruction. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(0), - calling_convention.GetFpuRegisterAt(1))); - locations->SetOut(Location::RegisterPairLocation(R0, R1)); - break; - } - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimChar: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to char is a result of code transformations. - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - // Processing a Dex `int-to-char' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimFloat: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-float' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimLong: { - // Processing a Dex `long-to-float' instruction. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); - locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); - break; - } - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-float' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - case Primitive::kPrimDouble: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-double' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimLong: - // Processing a Dex `long-to-double' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-double' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } -} - -void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { - LocationSummary* locations = conversion->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - Primitive::Type result_type = conversion->GetResultType(); - Primitive::Type input_type = conversion->GetInputType(); - DCHECK_NE(result_type, input_type); - switch (result_type) { - case Primitive::kPrimByte: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to byte is a result of code transformations. - __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); - break; - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-byte' instruction. - __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimShort: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to short is a result of code transformations. - __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); - break; - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-short' instruction. - __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimInt: - switch (input_type) { - case Primitive::kPrimLong: - // Processing a Dex `long-to-int' instruction. - DCHECK(out.IsRegister()); - if (in.IsRegisterPair()) { - __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); - } else if (in.IsDoubleStackSlot()) { - __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); - } else { - DCHECK(in.IsConstant()); - DCHECK(in.GetConstant()->IsLongConstant()); - int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); - __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); - } - break; - - case Primitive::kPrimFloat: { - // Processing a Dex `float-to-int' instruction. - SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - __ vcvtis(temp, in.AsFpuRegister<SRegister>()); - __ vmovrs(out.AsRegister<Register>(), temp); - break; - } - - case Primitive::kPrimDouble: { - // Processing a Dex `double-to-int' instruction. - SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - __ vmovrs(out.AsRegister<Register>(), temp_s); - break; - } - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimLong: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-long' instruction. - DCHECK(out.IsRegisterPair()); - DCHECK(in.IsRegister()); - __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); - // Sign extension. - __ Asr(out.AsRegisterPairHigh<Register>(), - out.AsRegisterPairLow<Register>(), - 31); - break; - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-long' instruction. - codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); - CheckEntrypointTypes<kQuickF2l, int64_t, float>(); - break; - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-long' instruction. - codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); - CheckEntrypointTypes<kQuickD2l, int64_t, double>(); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimChar: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to char is a result of code transformations. - __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); - break; - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - // Processing a Dex `int-to-char' instruction. - __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimFloat: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: { - // Processing a Dex `int-to-float' instruction. - __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); - __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimLong: - // Processing a Dex `long-to-float' instruction. - codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); - CheckEntrypointTypes<kQuickL2f, float, int64_t>(); - break; - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-float' instruction. - __ vcvtsd(out.AsFpuRegister<SRegister>(), - FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - case Primitive::kPrimDouble: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: { - // Processing a Dex `int-to-double' instruction. - __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); - __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - out.AsFpuRegisterPairLow<SRegister>()); - break; - } - - case Primitive::kPrimLong: { - // Processing a Dex `long-to-double' instruction. - Register low = in.AsRegisterPairLow<Register>(); - Register high = in.AsRegisterPairHigh<Register>(); - SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); - DRegister out_d = FromLowSToD(out_s); - SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - DRegister temp_d = FromLowSToD(temp_s); - SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); - DRegister constant_d = FromLowSToD(constant_s); - - // temp_d = int-to-double(high) - __ vmovsr(temp_s, high); - __ vcvtdi(temp_d, temp_s); - // constant_d = k2Pow32EncodingForDouble - __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); - // out_d = unsigned-to-double(low) - __ vmovsr(out_s, low); - __ vcvtdu(out_d, out_s); - // out_d += temp_d * constant_d - __ vmlad(out_d, temp_d, constant_d); - break; - } - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-double' instruction. - __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - in.AsFpuRegister<SRegister>()); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } -} - -void LocationsBuilderARM::VisitAdd(HAdd* add) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); - switch (add->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - - default: - LOG(FATAL) << "Unexpected add type " << add->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { - LocationSummary* locations = add->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - switch (add->GetResultType()) { - case Primitive::kPrimInt: - if (second.IsRegister()) { - __ add(out.AsRegister<Register>(), - first.AsRegister<Register>(), - ShifterOperand(second.AsRegister<Register>())); - } else { - __ AddConstant(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.GetConstant()->AsIntConstant()->GetValue()); - } - break; - - case Primitive::kPrimLong: { - if (second.IsConstant()) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); - GenerateAddLongConst(out, first, value); - } else { - DCHECK(second.IsRegisterPair()); - __ adds(out.AsRegisterPairLow<Register>(), - first.AsRegisterPairLow<Register>(), - ShifterOperand(second.AsRegisterPairLow<Register>())); - __ adc(out.AsRegisterPairHigh<Register>(), - first.AsRegisterPairHigh<Register>(), - ShifterOperand(second.AsRegisterPairHigh<Register>())); - } - break; - } - - case Primitive::kPrimFloat: - __ vadds(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - - case Primitive::kPrimDouble: - __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - - default: - LOG(FATAL) << "Unexpected add type " << add->GetResultType(); - } -} - -void LocationsBuilderARM::VisitSub(HSub* sub) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); - switch (sub->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - default: - LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { - LocationSummary* locations = sub->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - switch (sub->GetResultType()) { - case Primitive::kPrimInt: { - if (second.IsRegister()) { - __ sub(out.AsRegister<Register>(), - first.AsRegister<Register>(), - ShifterOperand(second.AsRegister<Register>())); - } else { - __ AddConstant(out.AsRegister<Register>(), - first.AsRegister<Register>(), - -second.GetConstant()->AsIntConstant()->GetValue()); - } - break; - } - - case Primitive::kPrimLong: { - if (second.IsConstant()) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); - GenerateAddLongConst(out, first, -value); - } else { - DCHECK(second.IsRegisterPair()); - __ subs(out.AsRegisterPairLow<Register>(), - first.AsRegisterPairLow<Register>(), - ShifterOperand(second.AsRegisterPairLow<Register>())); - __ sbc(out.AsRegisterPairHigh<Register>(), - first.AsRegisterPairHigh<Register>(), - ShifterOperand(second.AsRegisterPairHigh<Register>())); - } - break; - } - - case Primitive::kPrimFloat: { - __ vsubs(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimDouble: { - __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - } - - - default: - LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); - } -} - -void LocationsBuilderARM::VisitMul(HMul* mul) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); - switch (mul->GetResultType()) { - case Primitive::kPrimInt: - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - - default: - LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { - LocationSummary* locations = mul->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - switch (mul->GetResultType()) { - case Primitive::kPrimInt: { - __ mul(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.AsRegister<Register>()); - break; - } - case Primitive::kPrimLong: { - Register out_hi = out.AsRegisterPairHigh<Register>(); - Register out_lo = out.AsRegisterPairLow<Register>(); - Register in1_hi = first.AsRegisterPairHigh<Register>(); - Register in1_lo = first.AsRegisterPairLow<Register>(); - Register in2_hi = second.AsRegisterPairHigh<Register>(); - Register in2_lo = second.AsRegisterPairLow<Register>(); - - // Extra checks to protect caused by the existence of R1_R2. - // The algorithm is wrong if out.hi is either in1.lo or in2.lo: - // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); - DCHECK_NE(out_hi, in1_lo); - DCHECK_NE(out_hi, in2_lo); - - // input: in1 - 64 bits, in2 - 64 bits - // output: out - // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo - // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] - // parts: out.lo = (in1.lo * in2.lo)[31:0] - - // IP <- in1.lo * in2.hi - __ mul(IP, in1_lo, in2_hi); - // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo - __ mla(out_hi, in1_hi, in2_lo, IP); - // out.lo <- (in1.lo * in2.lo)[31:0]; - __ umull(out_lo, IP, in1_lo, in2_lo); - // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] - __ add(out_hi, out_hi, ShifterOperand(IP)); - break; - } - - case Primitive::kPrimFloat: { - __ vmuls(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimDouble: { - __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - } - - default: - LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - Register out = locations->Out().AsRegister<Register>(); - Register dividend = locations->InAt(0).AsRegister<Register>(); - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - DCHECK(imm == 1 || imm == -1); - - if (instruction->IsRem()) { - __ LoadImmediate(out, 0); - } else { - if (imm == 1) { - __ Mov(out, dividend); - } else { - __ rsb(out, dividend, ShifterOperand(0)); - } - } -} - -void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - Register out = locations->Out().AsRegister<Register>(); - Register dividend = locations->InAt(0).AsRegister<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); - int ctz_imm = CTZ(abs_imm); - - if (ctz_imm == 1) { - __ Lsr(temp, dividend, 32 - ctz_imm); - } else { - __ Asr(temp, dividend, 31); - __ Lsr(temp, temp, 32 - ctz_imm); - } - __ add(out, temp, ShifterOperand(dividend)); - - if (instruction->IsDiv()) { - __ Asr(out, out, ctz_imm); - if (imm < 0) { - __ rsb(out, out, ShifterOperand(0)); - } - } else { - __ ubfx(out, out, 0, ctz_imm); - __ sub(out, out, ShifterOperand(temp)); - } -} - -void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - Register out = locations->Out().AsRegister<Register>(); - Register dividend = locations->InAt(0).AsRegister<Register>(); - Register temp1 = locations->GetTemp(0).AsRegister<Register>(); - Register temp2 = locations->GetTemp(1).AsRegister<Register>(); - int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - - int64_t magic; - int shift; - CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); - - __ LoadImmediate(temp1, magic); - __ smull(temp2, temp1, dividend, temp1); - - if (imm > 0 && magic < 0) { - __ add(temp1, temp1, ShifterOperand(dividend)); - } else if (imm < 0 && magic > 0) { - __ sub(temp1, temp1, ShifterOperand(dividend)); - } - - if (shift != 0) { - __ Asr(temp1, temp1, shift); - } - - if (instruction->IsDiv()) { - __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); - } else { - __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); - // TODO: Strength reduction for mls. - __ LoadImmediate(temp2, imm); - __ mls(out, temp1, temp2, dividend); - } -} - -void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - if (imm == 0) { - // Do not generate anything. DivZeroCheck would prevent any code to be executed. - } else if (imm == 1 || imm == -1) { - DivRemOneOrMinusOne(instruction); - } else if (IsPowerOfTwo(AbsOrMin(imm))) { - DivRemByPowerOfTwo(instruction); - } else { - DCHECK(imm <= -2 || imm >= 2); - GenerateDivRemWithAnyConstant(instruction); - } -} - -void LocationsBuilderARM::VisitDiv(HDiv* div) { - LocationSummary::CallKind call_kind = LocationSummary::kNoCall; - if (div->GetResultType() == Primitive::kPrimLong) { - // pLdiv runtime call. - call_kind = LocationSummary::kCallOnMainOnly; - } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { - // sdiv will be replaced by other instruction sequence. - } else if (div->GetResultType() == Primitive::kPrimInt && - !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - // pIdivmod runtime call. - call_kind = LocationSummary::kCallOnMainOnly; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); - - switch (div->GetResultType()) { - case Primitive::kPrimInt: { - if (div->InputAt(1)->IsConstant()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); - if (value == 1 || value == 0 || value == -1) { - // No temp register required. - } else { - locations->AddTemp(Location::RequiresRegister()); - if (!IsPowerOfTwo(AbsOrMin(value))) { - locations->AddTemp(Location::RequiresRegister()); - } - } - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } else { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but - // we only need the former. - locations->SetOut(Location::RegisterLocation(R0)); - } - break; - } - case Primitive::kPrimLong: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); - locations->SetInAt(1, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); - locations->SetOut(Location::RegisterPairLocation(R0, R1)); - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - - default: - LOG(FATAL) << "Unexpected div type " << div->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { - LocationSummary* locations = div->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - switch (div->GetResultType()) { - case Primitive::kPrimInt: { - if (second.IsConstant()) { - GenerateDivRemConstantIntegral(div); - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - __ sdiv(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.AsRegister<Register>()); - } else { - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); - DCHECK_EQ(R0, out.AsRegister<Register>()); - - codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); - CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); - } - break; - } - - case Primitive::kPrimLong: { - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); - DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); - DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); - - codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); - CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); - break; - } - - case Primitive::kPrimFloat: { - __ vdivs(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimDouble: { - __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - } - - default: - LOG(FATAL) << "Unexpected div type " << div->GetResultType(); - } -} - -void LocationsBuilderARM::VisitRem(HRem* rem) { - Primitive::Type type = rem->GetResultType(); - - // Most remainders are implemented in the runtime. - LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; - if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { - // sdiv will be replaced by other instruction sequence. - call_kind = LocationSummary::kNoCall; - } else if ((rem->GetResultType() == Primitive::kPrimInt) - && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - // Have hardware divide instruction for int, do it with three instructions. - call_kind = LocationSummary::kNoCall; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); - - switch (type) { - case Primitive::kPrimInt: { - if (rem->InputAt(1)->IsConstant()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); - if (value == 1 || value == 0 || value == -1) { - // No temp register required. - } else { - locations->AddTemp(Location::RequiresRegister()); - if (!IsPowerOfTwo(AbsOrMin(value))) { - locations->AddTemp(Location::RequiresRegister()); - } - } - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - locations->AddTemp(Location::RequiresRegister()); - } else { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but - // we only need the latter. - locations->SetOut(Location::RegisterLocation(R1)); - } - break; - } - case Primitive::kPrimLong: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); - locations->SetInAt(1, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); - // The runtime helper puts the output in R2,R3. - locations->SetOut(Location::RegisterPairLocation(R2, R3)); - break; - } - case Primitive::kPrimFloat: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); - locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); - locations->SetOut(Location::FpuRegisterLocation(S0)); - break; - } - - case Primitive::kPrimDouble: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); - locations->SetInAt(1, Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); - locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); - break; - } - - default: - LOG(FATAL) << "Unexpected rem type " << type; - } -} - -void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { - LocationSummary* locations = rem->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - Primitive::Type type = rem->GetResultType(); - switch (type) { - case Primitive::kPrimInt: { - if (second.IsConstant()) { - GenerateDivRemConstantIntegral(rem); - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - Register reg1 = first.AsRegister<Register>(); - Register reg2 = second.AsRegister<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - - // temp = reg1 / reg2 (integer division) - // dest = reg1 - temp * reg2 - __ sdiv(temp, reg1, reg2); - __ mls(out.AsRegister<Register>(), temp, reg2, reg1); - } else { - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); - DCHECK_EQ(R1, out.AsRegister<Register>()); - - codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); - } - break; - } - - case Primitive::kPrimLong: { - codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); - break; - } - - case Primitive::kPrimFloat: { - codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickFmodf, float, float, float>(); - break; - } - - case Primitive::kPrimDouble: { - codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickFmod, double, double, double>(); - break; - } - - default: - LOG(FATAL) << "Unexpected rem type " << type; - } -} - -void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { - LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); - locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); -} - -void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); - codegen_->AddSlowPath(slow_path); - - LocationSummary* locations = instruction->GetLocations(); - Location value = locations->InAt(0); - - switch (instruction->GetType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: { - if (value.IsRegister()) { - __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); - } else { - DCHECK(value.IsConstant()) << value; - if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { - __ b(slow_path->GetEntryLabel()); - } - } - break; - } - case Primitive::kPrimLong: { - if (value.IsRegisterPair()) { - __ orrs(IP, - value.AsRegisterPairLow<Register>(), - ShifterOperand(value.AsRegisterPairHigh<Register>())); - __ b(slow_path->GetEntryLabel(), EQ); - } else { - DCHECK(value.IsConstant()) << value; - if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { - __ b(slow_path->GetEntryLabel()); - } - } - break; - default: - LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); - } - } -} - -void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { - Register in = locations->InAt(0).AsRegister<Register>(); - Location rhs = locations->InAt(1); - Register out = locations->Out().AsRegister<Register>(); - - if (rhs.IsConstant()) { - // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], - // so map all rotations to a +ve. equivalent in that range. - // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) - uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; - if (rot) { - // Rotate, mapping left rotations to right equivalents if necessary. - // (e.g. left by 2 bits == right by 30.) - __ Ror(out, in, rot); - } else if (out != in) { - __ Mov(out, in); - } - } else { - __ Ror(out, in, rhs.AsRegister<Register>()); - } -} - -// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer -// rotates by swapping input regs (effectively rotating by the first 32-bits of -// a larger rotation) or flipping direction (thus treating larger right/left -// rotations as sub-word sized rotations in the other direction) as appropriate. -void InstructionCodeGeneratorARM::HandleLongRotate(HRor* ror) { - LocationSummary* locations = ror->GetLocations(); - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Location rhs = locations->InAt(1); - Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); - - if (rhs.IsConstant()) { - uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); - // Map all rotations to +ve. equivalents on the interval [0,63]. - rot &= kMaxLongShiftDistance; - // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate - // logic below to a simple pair of binary orr. - // (e.g. 34 bits == in_reg swap + 2 bits right.) - if (rot >= kArmBitsPerWord) { - rot -= kArmBitsPerWord; - std::swap(in_reg_hi, in_reg_lo); - } - // Rotate, or mov to out for zero or word size rotations. - if (rot != 0u) { - __ Lsr(out_reg_hi, in_reg_hi, rot); - __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); - __ Lsr(out_reg_lo, in_reg_lo, rot); - __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); - } else { - __ Mov(out_reg_lo, in_reg_lo); - __ Mov(out_reg_hi, in_reg_hi); - } - } else { - Register shift_right = locations->GetTemp(0).AsRegister<Register>(); - Register shift_left = locations->GetTemp(1).AsRegister<Register>(); - Label end; - Label shift_by_32_plus_shift_right; - Label* final_label = codegen_->GetFinalLabel(ror, &end); - - __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); - __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); - __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); - __ b(&shift_by_32_plus_shift_right, CC); - - // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). - // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). - __ Lsl(out_reg_hi, in_reg_hi, shift_left); - __ Lsr(out_reg_lo, in_reg_lo, shift_right); - __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); - __ Lsl(out_reg_lo, in_reg_lo, shift_left); - __ Lsr(shift_left, in_reg_hi, shift_right); - __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); - __ b(final_label); - - __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. - // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). - // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). - __ Lsr(out_reg_hi, in_reg_hi, shift_right); - __ Lsl(out_reg_lo, in_reg_lo, shift_left); - __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); - __ Lsr(out_reg_lo, in_reg_lo, shift_right); - __ Lsl(shift_right, in_reg_hi, shift_left); - __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); - - if (end.IsLinked()) { - __ Bind(&end); - } - } -} - -void LocationsBuilderARM::VisitRor(HRor* ror) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); - switch (ror->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - if (ror->InputAt(1)->IsConstant()) { - locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - } - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { - LocationSummary* locations = ror->GetLocations(); - Primitive::Type type = ror->GetResultType(); - switch (type) { - case Primitive::kPrimInt: { - HandleIntegerRotate(locations); - break; - } - case Primitive::kPrimLong: { - HandleLongRotate(ror); - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << type; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { - DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); - - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); - - switch (op->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - if (op->InputAt(1)->IsConstant()) { - locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - // Make the output overlap, as it will be used to hold the masked - // second input. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - } - break; - } - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - if (op->InputAt(1)->IsConstant()) { - locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); - // For simplicity, use kOutputOverlap even though we only require that low registers - // don't clash with high registers which the register allocator currently guarantees. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - } - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { - DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); - - LocationSummary* locations = op->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - Primitive::Type type = op->GetResultType(); - switch (type) { - case Primitive::kPrimInt: { - Register out_reg = out.AsRegister<Register>(); - Register first_reg = first.AsRegister<Register>(); - if (second.IsRegister()) { - Register second_reg = second.AsRegister<Register>(); - // ARM doesn't mask the shift count so we need to do it ourselves. - __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); - if (op->IsShl()) { - __ Lsl(out_reg, first_reg, out_reg); - } else if (op->IsShr()) { - __ Asr(out_reg, first_reg, out_reg); - } else { - __ Lsr(out_reg, first_reg, out_reg); - } - } else { - int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t shift_value = cst & kMaxIntShiftDistance; - if (shift_value == 0) { // ARM does not support shifting with 0 immediate. - __ Mov(out_reg, first_reg); - } else if (op->IsShl()) { - __ Lsl(out_reg, first_reg, shift_value); - } else if (op->IsShr()) { - __ Asr(out_reg, first_reg, shift_value); - } else { - __ Lsr(out_reg, first_reg, shift_value); - } - } - break; - } - case Primitive::kPrimLong: { - Register o_h = out.AsRegisterPairHigh<Register>(); - Register o_l = out.AsRegisterPairLow<Register>(); - - Register high = first.AsRegisterPairHigh<Register>(); - Register low = first.AsRegisterPairLow<Register>(); - - if (second.IsRegister()) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - - Register second_reg = second.AsRegister<Register>(); - - if (op->IsShl()) { - __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); - // Shift the high part - __ Lsl(o_h, high, o_l); - // Shift the low part and `or` what overflew on the high part - __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); - __ Lsr(temp, low, temp); - __ orr(o_h, o_h, ShifterOperand(temp)); - // If the shift is > 32 bits, override the high part - __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); - __ it(PL); - __ Lsl(o_h, low, temp, PL); - // Shift the low part - __ Lsl(o_l, low, o_l); - } else if (op->IsShr()) { - __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); - // Shift the low part - __ Lsr(o_l, low, o_h); - // Shift the high part and `or` what underflew on the low part - __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ Lsl(temp, high, temp); - __ orr(o_l, o_l, ShifterOperand(temp)); - // If the shift is > 32 bits, override the low part - __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ it(PL); - __ Asr(o_l, high, temp, PL); - // Shift the high part - __ Asr(o_h, high, o_h); - } else { - __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); - // same as Shr except we use `Lsr`s and not `Asr`s - __ Lsr(o_l, low, o_h); - __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ Lsl(temp, high, temp); - __ orr(o_l, o_l, ShifterOperand(temp)); - __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ it(PL); - __ Lsr(o_l, high, temp, PL); - __ Lsr(o_h, high, o_h); - } - } else { - // Register allocator doesn't create partial overlap. - DCHECK_NE(o_l, high); - DCHECK_NE(o_h, low); - int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t shift_value = cst & kMaxLongShiftDistance; - if (shift_value > 32) { - if (op->IsShl()) { - __ Lsl(o_h, low, shift_value - 32); - __ LoadImmediate(o_l, 0); - } else if (op->IsShr()) { - __ Asr(o_l, high, shift_value - 32); - __ Asr(o_h, high, 31); - } else { - __ Lsr(o_l, high, shift_value - 32); - __ LoadImmediate(o_h, 0); - } - } else if (shift_value == 32) { - if (op->IsShl()) { - __ mov(o_h, ShifterOperand(low)); - __ LoadImmediate(o_l, 0); - } else if (op->IsShr()) { - __ mov(o_l, ShifterOperand(high)); - __ Asr(o_h, high, 31); - } else { - __ mov(o_l, ShifterOperand(high)); - __ LoadImmediate(o_h, 0); - } - } else if (shift_value == 1) { - if (op->IsShl()) { - __ Lsls(o_l, low, 1); - __ adc(o_h, high, ShifterOperand(high)); - } else if (op->IsShr()) { - __ Asrs(o_h, high, 1); - __ Rrx(o_l, low); - } else { - __ Lsrs(o_h, high, 1); - __ Rrx(o_l, low); - } - } else { - DCHECK(2 <= shift_value && shift_value < 32) << shift_value; - if (op->IsShl()) { - __ Lsl(o_h, high, shift_value); - __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); - __ Lsl(o_l, low, shift_value); - } else if (op->IsShr()) { - __ Lsr(o_l, low, shift_value); - __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); - __ Asr(o_h, high, shift_value); - } else { - __ Lsr(o_l, low, shift_value); - __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); - __ Lsr(o_h, high, shift_value); - } - } - } - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << type; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitShl(HShl* shl) { - HandleShift(shl); -} - -void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { - HandleShift(shl); -} - -void LocationsBuilderARM::VisitShr(HShr* shr) { - HandleShift(shr); -} - -void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { - HandleShift(shr); -} - -void LocationsBuilderARM::VisitUShr(HUShr* ushr) { - HandleShift(ushr); -} - -void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { - HandleShift(ushr); -} - -void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - if (instruction->IsStringAlloc()) { - locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); - } else { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - } - locations->SetOut(Location::RegisterLocation(R0)); -} - -void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { - // Note: if heap poisoning is enabled, the entry point takes cares - // of poisoning the reference. - if (instruction->IsStringAlloc()) { - // String is allocated through StringFactory. Call NewEmptyString entry point. - Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); - MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); - __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); - __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); - __ blx(LR); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); - } else { - codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); - } -} - -void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - InvokeRuntimeCallingConvention calling_convention; - locations->SetOut(Location::RegisterLocation(R0)); - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); -} - -void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { - // Note: if heap poisoning is enabled, the entry point takes cares - // of poisoning the reference. - QuickEntrypointEnum entrypoint = - CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); - codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); - DCHECK(!codegen_->IsLeafMethod()); -} - -void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); - if (location.IsStackSlot()) { - location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); - } else if (location.IsDoubleStackSlot()) { - location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); - } - locations->SetOut(location); -} - -void InstructionCodeGeneratorARM::VisitParameterValue( - HParameterValue* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, the parameter is already at its location. -} - -void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); -} - -void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, the method is already at its location. -} - -void LocationsBuilderARM::VisitNot(HNot* not_) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { - LocationSummary* locations = not_->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - switch (not_->GetResultType()) { - case Primitive::kPrimInt: - __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); - break; - - case Primitive::kPrimLong: - __ mvn(out.AsRegisterPairLow<Register>(), - ShifterOperand(in.AsRegisterPairLow<Register>())); - __ mvn(out.AsRegisterPairHigh<Register>(), - ShifterOperand(in.AsRegisterPairHigh<Register>())); - break; - - default: - LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); - } -} - -void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { - LocationSummary* locations = bool_not->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); -} - -void LocationsBuilderARM::VisitCompare(HCompare* compare) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); - switch (compare->InputAt(0)->GetType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - // Output overlaps because it is written before doing the low comparison. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); - locations->SetOut(Location::RequiresRegister()); - break; - } - default: - LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); - } -} - -void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { - LocationSummary* locations = compare->GetLocations(); - Register out = locations->Out().AsRegister<Register>(); - Location left = locations->InAt(0); - Location right = locations->InAt(1); - - Label less, greater, done; - Label* final_label = codegen_->GetFinalLabel(compare, &done); - Primitive::Type type = compare->InputAt(0)->GetType(); - Condition less_cond; - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: { - __ LoadImmediate(out, 0); - __ cmp(left.AsRegister<Register>(), - ShifterOperand(right.AsRegister<Register>())); // Signed compare. - less_cond = LT; - break; - } - case Primitive::kPrimLong: { - __ cmp(left.AsRegisterPairHigh<Register>(), - ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. - __ b(&less, LT); - __ b(&greater, GT); - // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. - __ LoadImmediate(out, 0); - __ cmp(left.AsRegisterPairLow<Register>(), - ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. - less_cond = LO; - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - __ LoadImmediate(out, 0); - GenerateVcmp(compare, codegen_); - __ vmstat(); // transfer FP status register to ARM APSR. - less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); - break; - } - default: - LOG(FATAL) << "Unexpected compare type " << type; - UNREACHABLE(); - } - - __ b(final_label, EQ); - __ b(&less, less_cond); - - __ Bind(&greater); - __ LoadImmediate(out, 1); - __ b(final_label); - - __ Bind(&less); - __ LoadImmediate(out, -1); - - if (done.IsLinked()) { - __ Bind(&done); - } -} - -void LocationsBuilderARM::VisitPhi(HPhi* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { - locations->SetInAt(i, Location::Any()); - } - locations->SetOut(Location::Any()); -} - -void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { - LOG(FATAL) << "Unreachable"; -} - -void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { - // TODO (ported from quick): revisit ARM barrier kinds. - DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. - switch (kind) { - case MemBarrierKind::kAnyStore: - case MemBarrierKind::kLoadAny: - case MemBarrierKind::kAnyAny: { - flavor = DmbOptions::ISH; - break; - } - case MemBarrierKind::kStoreStore: { - flavor = DmbOptions::ISHST; - break; - } - default: - LOG(FATAL) << "Unexpected memory barrier " << kind; - } - __ dmb(flavor); -} - -void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, - uint32_t offset, - Register out_lo, - Register out_hi) { - if (offset != 0) { - // Ensure `out_lo` is different from `addr`, so that loading - // `offset` into `out_lo` does not clutter `addr`. - DCHECK_NE(out_lo, addr); - __ LoadImmediate(out_lo, offset); - __ add(IP, addr, ShifterOperand(out_lo)); - addr = IP; - } - __ ldrexd(out_lo, out_hi, addr); -} - -void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, - uint32_t offset, - Register value_lo, - Register value_hi, - Register temp1, - Register temp2, - HInstruction* instruction) { - Label fail; - if (offset != 0) { - __ LoadImmediate(temp1, offset); - __ add(IP, addr, ShifterOperand(temp1)); - addr = IP; - } - __ Bind(&fail); - // We need a load followed by store. (The address used in a STREX instruction must - // be the same as the address in the most recently executed LDREX instruction.) - __ ldrexd(temp1, temp2, addr); - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ strexd(temp1, value_lo, value_hi, addr); - __ CompareAndBranchIfNonZero(temp1, &fail); -} - -void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { - DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); - - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - - Primitive::Type field_type = field_info.GetFieldType(); - if (Primitive::IsFloatingPointType(field_type)) { - locations->SetInAt(1, Location::RequiresFpuRegister()); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - } - - bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; - bool generate_volatile = field_info.IsVolatile() - && is_wide - && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); - // Temporary registers for the write barrier. - // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. - if (needs_write_barrier) { - locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. - locations->AddTemp(Location::RequiresRegister()); - } else if (generate_volatile) { - // ARM encoding have some additional constraints for ldrexd/strexd: - // - registers need to be consecutive - // - the first register should be even but not R14. - // We don't test for ARM yet, and the assertion makes sure that we - // revisit this if we ever enable ARM encoding. - DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); - - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - if (field_type == Primitive::kPrimDouble) { - // For doubles we need two more registers to copy the value. - locations->AddTemp(Location::RegisterLocation(R2)); - locations->AddTemp(Location::RegisterLocation(R3)); - } - } -} - -void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, - const FieldInfo& field_info, - bool value_can_be_null) { - DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); - - LocationSummary* locations = instruction->GetLocations(); - Register base = locations->InAt(0).AsRegister<Register>(); - Location value = locations->InAt(1); - - bool is_volatile = field_info.IsVolatile(); - bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - Primitive::Type field_type = field_info.GetFieldType(); - uint32_t offset = field_info.GetFieldOffset().Uint32Value(); - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); - - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); - } - - switch (field_type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: { - __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); - break; - } - - case Primitive::kPrimShort: - case Primitive::kPrimChar: { - __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); - break; - } - - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - if (kPoisonHeapReferences && needs_write_barrier) { - // Note that in the case where `value` is a null reference, - // we do not enter this block, as a null reference does not - // need poisoning. - DCHECK_EQ(field_type, Primitive::kPrimNot); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - __ Mov(temp, value.AsRegister<Register>()); - __ PoisonHeapReference(temp); - __ StoreToOffset(kStoreWord, temp, base, offset); - } else { - __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); - } - break; - } - - case Primitive::kPrimLong: { - if (is_volatile && !atomic_ldrd_strd) { - GenerateWideAtomicStore(base, offset, - value.AsRegisterPairLow<Register>(), - value.AsRegisterPairHigh<Register>(), - locations->GetTemp(0).AsRegister<Register>(), - locations->GetTemp(1).AsRegister<Register>(), - instruction); - } else { - __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - break; - } - - case Primitive::kPrimFloat: { - __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); - break; - } - - case Primitive::kPrimDouble: { - DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); - if (is_volatile && !atomic_ldrd_strd) { - Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); - Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); - - __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); - - GenerateWideAtomicStore(base, offset, - value_reg_lo, - value_reg_hi, - locations->GetTemp(2).AsRegister<Register>(), - locations->GetTemp(3).AsRegister<Register>(), - instruction); - } else { - __ StoreDToOffset(value_reg, base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << field_type; - UNREACHABLE(); - } - - // Longs and doubles are handled in the switch. - if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - - if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register card = locations->GetTemp(1).AsRegister<Register>(); - codegen_->MarkGCCard( - temp, card, base, value.AsRegister<Register>(), value_can_be_null); - } - - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); - } -} - -void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { - DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); - - bool object_field_get_with_read_barrier = - kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, - object_field_get_with_read_barrier ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall); - if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::RequiresRegister()); - - bool volatile_for_double = field_info.IsVolatile() - && (field_info.GetFieldType() == Primitive::kPrimDouble) - && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - // The output overlaps in case of volatile long: we don't want the - // code generated by GenerateWideAtomicLoad to overwrite the - // object's location. Likewise, in the case of an object field get - // with read barriers enabled, we do not want the load to overwrite - // the object's location, as we need it to emit the read barrier. - bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || - object_field_get_with_read_barrier; - - if (Primitive::IsFloatingPointType(instruction->GetType())) { - locations->SetOut(Location::RequiresFpuRegister()); - } else { - locations->SetOut(Location::RequiresRegister(), - (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); - } - if (volatile_for_double) { - // ARM encoding have some additional constraints for ldrexd/strexd: - // - registers need to be consecutive - // - the first register should be even but not R14. - // We don't test for ARM yet, and the assertion makes sure that we - // revisit this if we ever enable ARM encoding. - DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { - // We need a temporary register for the read barrier marking slow - // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. - if (kBakerReadBarrierLinkTimeThunksEnableForFields && - !Runtime::Current()->UseJitCompilation()) { - // If link-time thunks for the Baker read barrier are enabled, for AOT - // loads we need a temporary only if the offset is too big. - if (field_info.GetFieldOffset().Uint32Value() >= kReferenceLoadMinFarOffset) { - locations->AddTemp(Location::RequiresRegister()); - } - // And we always need the reserved entrypoint register. - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } else { - locations->AddTemp(Location::RequiresRegister()); - } - } -} - -Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { - DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) - << input->GetType(); - if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || - (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { - return Location::ConstantLocation(input->AsConstant()); - } else { - return Location::RequiresFpuRegister(); - } -} - -Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, - Opcode opcode) { - DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); - if (constant->IsConstant() && - CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { - return Location::ConstantLocation(constant->AsConstant()); - } - return Location::RequiresRegister(); -} - -bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, - Opcode opcode) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); - if (Primitive::Is64BitType(input_cst->GetType())) { - Opcode high_opcode = opcode; - SetCc low_set_cc = kCcDontCare; - switch (opcode) { - case SUB: - // Flip the operation to an ADD. - value = -value; - opcode = ADD; - FALLTHROUGH_INTENDED; - case ADD: - if (Low32Bits(value) == 0u) { - return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); - } - high_opcode = ADC; - low_set_cc = kCcSet; - break; - default: - break; - } - return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && - CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); - } else { - return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); - } -} - -bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, - Opcode opcode, - SetCc set_cc) { - ShifterOperand so; - ArmAssembler* assembler = codegen_->GetAssembler(); - if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { - return true; - } - Opcode neg_opcode = kNoOperand; - uint32_t neg_value = 0; - switch (opcode) { - case AND: neg_opcode = BIC; neg_value = ~value; break; - case ORR: neg_opcode = ORN; neg_value = ~value; break; - case ADD: neg_opcode = SUB; neg_value = -value; break; - case ADC: neg_opcode = SBC; neg_value = ~value; break; - case SUB: neg_opcode = ADD; neg_value = -value; break; - case SBC: neg_opcode = ADC; neg_value = ~value; break; - case MOV: neg_opcode = MVN; neg_value = ~value; break; - default: - return false; - } - - if (assembler->ShifterOperandCanHold(kNoRegister, - kNoRegister, - neg_opcode, - neg_value, - set_cc, - &so)) { - return true; - } - - return opcode == AND && IsPowerOfTwo(value + 1); -} - -void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, - const FieldInfo& field_info) { - DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); - - LocationSummary* locations = instruction->GetLocations(); - Location base_loc = locations->InAt(0); - Register base = base_loc.AsRegister<Register>(); - Location out = locations->Out(); - bool is_volatile = field_info.IsVolatile(); - bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - Primitive::Type field_type = field_info.GetFieldType(); - uint32_t offset = field_info.GetFieldOffset().Uint32Value(); - - switch (field_type) { - case Primitive::kPrimBoolean: - __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimByte: - __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimShort: - __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimChar: - __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimInt: - __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimNot: { - // /* HeapReference<Object> */ out = *(base + offset) - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - Location temp_loc = locations->GetTemp(0); - // Note that a potential implicit null check is handled in this - // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. - codegen_->GenerateFieldLoadWithBakerReadBarrier( - instruction, out, base, offset, temp_loc, /* needs_null_check */ true); - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); - } - } else { - __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); - } - // If read barriers are enabled, emit read barriers other than - // Baker's using a slow path (and also unpoison the loaded - // reference, if heap poisoning is enabled). - codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); - } - break; - } - - case Primitive::kPrimLong: - if (is_volatile && !atomic_ldrd_strd) { - GenerateWideAtomicLoad(base, offset, - out.AsRegisterPairLow<Register>(), - out.AsRegisterPairHigh<Register>()); - } else { - __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); - } - break; - - case Primitive::kPrimFloat: - __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); - break; - - case Primitive::kPrimDouble: { - DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); - if (is_volatile && !atomic_ldrd_strd) { - Register lo = locations->GetTemp(0).AsRegister<Register>(); - Register hi = locations->GetTemp(1).AsRegister<Register>(); - GenerateWideAtomicLoad(base, offset, lo, hi); - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ vmovdrr(out_reg, lo, hi); - } else { - __ LoadDFromOffset(out_reg, base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << field_type; - UNREACHABLE(); - } - - if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { - // Potential implicit null checks, in the case of reference or - // double fields, are handled in the previous switch statement. - } else { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - - if (is_volatile) { - if (field_type == Primitive::kPrimNot) { - // Memory barriers, in the case of references, are also handled - // in the previous switch statement. - } else { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); - } - } -} - -void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); -} - -void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); -} - -void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( - HUnresolvedInstanceFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( - HUnresolvedInstanceFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( - HUnresolvedInstanceFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( - HUnresolvedInstanceFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( - HUnresolvedStaticFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( - HUnresolvedStaticFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( - HUnresolvedStaticFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( - HUnresolvedStaticFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { - LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); - locations->SetInAt(0, Location::RequiresRegister()); -} - -void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { - if (CanMoveNullCheckToUser(instruction)) { - return; - } - Location obj = instruction->GetLocations()->InAt(0); - - __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); - RecordPcInfo(instruction, instruction->GetDexPc()); -} - -void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); - AddSlowPath(slow_path); - - LocationSummary* locations = instruction->GetLocations(); - Location obj = locations->InAt(0); - - __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); -} - -void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { - codegen_->GenerateNullCheck(instruction); -} - -static LoadOperandType GetLoadOperandType(Primitive::Type type) { - switch (type) { - case Primitive::kPrimNot: - return kLoadWord; - case Primitive::kPrimBoolean: - return kLoadUnsignedByte; - case Primitive::kPrimByte: - return kLoadSignedByte; - case Primitive::kPrimChar: - return kLoadUnsignedHalfword; - case Primitive::kPrimShort: - return kLoadSignedHalfword; - case Primitive::kPrimInt: - return kLoadWord; - case Primitive::kPrimLong: - return kLoadWordPair; - case Primitive::kPrimFloat: - return kLoadSWord; - case Primitive::kPrimDouble: - return kLoadDWord; - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -static StoreOperandType GetStoreOperandType(Primitive::Type type) { - switch (type) { - case Primitive::kPrimNot: - return kStoreWord; - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - return kStoreByte; - case Primitive::kPrimChar: - case Primitive::kPrimShort: - return kStoreHalfword; - case Primitive::kPrimInt: - return kStoreWord; - case Primitive::kPrimLong: - return kStoreWordPair; - case Primitive::kPrimFloat: - return kStoreSWord; - case Primitive::kPrimDouble: - return kStoreDWord; - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, - Location out_loc, - Register base, - Register reg_offset, - Condition cond) { - uint32_t shift_count = Primitive::ComponentSizeShift(type); - Address mem_address(base, reg_offset, Shift::LSL, shift_count); - - switch (type) { - case Primitive::kPrimByte: - __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimBoolean: - __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimShort: - __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimChar: - __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimNot: - case Primitive::kPrimInt: - __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); - break; - // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. - case Primitive::kPrimLong: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, - Location loc, - Register base, - Register reg_offset, - Condition cond) { - uint32_t shift_count = Primitive::ComponentSizeShift(type); - Address mem_address(base, reg_offset, Shift::LSL, shift_count); - - switch (type) { - case Primitive::kPrimByte: - case Primitive::kPrimBoolean: - __ strb(loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimShort: - case Primitive::kPrimChar: - __ strh(loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimNot: - case Primitive::kPrimInt: - __ str(loc.AsRegister<Register>(), mem_address, cond); - break; - // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. - case Primitive::kPrimLong: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { - bool object_array_get_with_read_barrier = - kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, - object_array_get_with_read_barrier ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall); - if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(instruction->GetType())) { - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - } else { - // The output overlaps in the case of an object array get with - // read barriers enabled: we do not want the move to overwrite the - // array's location, as we need it to emit the read barrier. - locations->SetOut( - Location::RequiresRegister(), - object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); - } - if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { - // We need a temporary register for the read barrier marking slow - // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. - if (kBakerReadBarrierLinkTimeThunksEnableForFields && - !Runtime::Current()->UseJitCompilation() && - instruction->GetIndex()->IsConstant()) { - // Array loads with constant index are treated as field loads. - // If link-time thunks for the Baker read barrier are enabled, for AOT - // constant index loads we need a temporary only if the offset is too big. - uint32_t offset = CodeGenerator::GetArrayDataOffset(instruction); - uint32_t index = instruction->GetIndex()->AsIntConstant()->GetValue(); - offset += index << Primitive::ComponentSizeShift(Primitive::kPrimNot); - if (offset >= kReferenceLoadMinFarOffset) { - locations->AddTemp(Location::RequiresRegister()); - } - // And we always need the reserved entrypoint register. - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } else if (kBakerReadBarrierLinkTimeThunksEnableForArrays && - !Runtime::Current()->UseJitCompilation() && - !instruction->GetIndex()->IsConstant()) { - // We need a non-scratch temporary for the array data pointer. - locations->AddTemp(Location::RequiresRegister()); - // And we always need the reserved entrypoint register. - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } else { - locations->AddTemp(Location::RequiresRegister()); - } - } else if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { - // Also need a temporary for String compression feature. - locations->AddTemp(Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location obj_loc = locations->InAt(0); - Register obj = obj_loc.AsRegister<Register>(); - Location index = locations->InAt(1); - Location out_loc = locations->Out(); - uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); - Primitive::Type type = instruction->GetType(); - const bool maybe_compressed_char_at = mirror::kUseStringCompression && - instruction->IsStringCharAt(); - HInstruction* array_instr = instruction->GetArray(); - bool has_intermediate_address = array_instr->IsIntermediateAddress(); - - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: { - Register length; - if (maybe_compressed_char_at) { - length = locations->GetTemp(0).AsRegister<Register>(); - uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); - __ LoadFromOffset(kLoadWord, length, obj, count_offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - if (index.IsConstant()) { - int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); - if (maybe_compressed_char_at) { - Label uncompressed_load, done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ b(&uncompressed_load, CS); - __ LoadFromOffset(kLoadUnsignedByte, - out_loc.AsRegister<Register>(), - obj, - data_offset + const_index); - __ b(final_label); - __ Bind(&uncompressed_load); - __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), - out_loc.AsRegister<Register>(), - obj, - data_offset + (const_index << 1)); - if (done.IsLinked()) { - __ Bind(&done); - } - } else { - uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); - - LoadOperandType load_type = GetLoadOperandType(type); - __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); - } - } else { - Register temp = IP; - - if (has_intermediate_address) { - // We do not need to compute the intermediate address from the array: the - // input instruction has done it already. See the comment in - // `TryExtractArrayAccessAddress()`. - if (kIsDebugBuild) { - HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); - } - temp = obj; - } else { - __ add(temp, obj, ShifterOperand(data_offset)); - } - if (maybe_compressed_char_at) { - Label uncompressed_load, done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ b(&uncompressed_load, CS); - __ ldrb(out_loc.AsRegister<Register>(), - Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); - __ b(final_label); - __ Bind(&uncompressed_load); - __ ldrh(out_loc.AsRegister<Register>(), - Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); - if (done.IsLinked()) { - __ Bind(&done); - } - } else { - codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); - } - } - break; - } - - case Primitive::kPrimNot: { - // The read barrier instrumentation of object ArrayGet - // instructions does not support the HIntermediateAddress - // instruction. - DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); - - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - // /* HeapReference<Object> */ out = - // *(obj + data_offset + index * sizeof(HeapReference<Object>)) - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - Location temp = locations->GetTemp(0); - // Note that a potential implicit null check is handled in this - // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. - DCHECK(!instruction->CanDoImplicitNullCheckOn(instruction->InputAt(0))); - if (index.IsConstant()) { - // Array load with a constant index can be treated as a field load. - data_offset += helpers::Int32ConstantFrom(index) << Primitive::ComponentSizeShift(type); - codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, - out_loc, - obj, - data_offset, - locations->GetTemp(0), - /* needs_null_check */ false); - } else { - codegen_->GenerateArrayLoadWithBakerReadBarrier( - instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ false); - } - } else { - Register out = out_loc.AsRegister<Register>(); - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ LoadFromOffset(kLoadWord, out, obj, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - // If read barriers are enabled, emit read barriers other than - // Baker's using a slow path (and also unpoison the loaded - // reference, if heap poisoning is enabled). - codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); - } else { - Register temp = IP; - - if (has_intermediate_address) { - // We do not need to compute the intermediate address from the array: the - // input instruction has done it already. See the comment in - // `TryExtractArrayAccessAddress()`. - if (kIsDebugBuild) { - HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); - } - temp = obj; - } else { - __ add(temp, obj, ShifterOperand(data_offset)); - } - codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); - - codegen_->MaybeRecordImplicitNullCheck(instruction); - // If read barriers are enabled, emit read barriers other than - // Baker's using a slow path (and also unpoison the loaded - // reference, if heap poisoning is enabled). - codegen_->MaybeGenerateReadBarrierSlow( - instruction, out_loc, out_loc, obj_loc, data_offset, index); - } - } - break; - } - - case Primitive::kPrimLong: { - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); - } else { - __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); - } - break; - } - - case Primitive::kPrimFloat: { - SRegister out = out_loc.AsFpuRegister<SRegister>(); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ LoadSFromOffset(out, obj, offset); - } else { - __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); - __ LoadSFromOffset(out, IP, data_offset); - } - break; - } - - case Primitive::kPrimDouble: { - SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ LoadDFromOffset(FromLowSToD(out), obj, offset); - } else { - __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); - } - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } - - if (type == Primitive::kPrimNot) { - // Potential implicit null checks, in the case of reference - // arrays, are handled in the previous switch statement. - } else if (!maybe_compressed_char_at) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } -} - -void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { - Primitive::Type value_type = instruction->GetComponentType(); - - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); - bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( - instruction, - may_need_runtime_call_for_type_check ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(value_type)) { - locations->SetInAt(2, Location::RequiresFpuRegister()); - } else { - locations->SetInAt(2, Location::RequiresRegister()); - } - if (needs_write_barrier) { - // Temporary registers for the write barrier. - locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. - locations->AddTemp(Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location array_loc = locations->InAt(0); - Register array = array_loc.AsRegister<Register>(); - Location index = locations->InAt(1); - Primitive::Type value_type = instruction->GetComponentType(); - bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); - uint32_t data_offset = - mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); - Location value_loc = locations->InAt(2); - HInstruction* array_instr = instruction->GetArray(); - bool has_intermediate_address = array_instr->IsIntermediateAddress(); - - switch (value_type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: { - if (index.IsConstant()) { - int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); - uint32_t full_offset = - data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); - StoreOperandType store_type = GetStoreOperandType(value_type); - __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); - } else { - Register temp = IP; - - if (has_intermediate_address) { - // We do not need to compute the intermediate address from the array: the - // input instruction has done it already. See the comment in - // `TryExtractArrayAccessAddress()`. - if (kIsDebugBuild) { - HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); - } - temp = array; - } else { - __ add(temp, array, ShifterOperand(data_offset)); - } - codegen_->StoreToShiftedRegOffset(value_type, - value_loc, - temp, - index.AsRegister<Register>()); - } - break; - } - - case Primitive::kPrimNot: { - Register value = value_loc.AsRegister<Register>(); - // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. - // See the comment in instruction_simplifier_shared.cc. - DCHECK(!has_intermediate_address); - - if (instruction->InputAt(2)->IsNullConstant()) { - // Just setting null. - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreToOffset(kStoreWord, value, array, offset); - } else { - DCHECK(index.IsRegister()) << index; - __ add(IP, array, ShifterOperand(data_offset)); - codegen_->StoreToShiftedRegOffset(value_type, - value_loc, - IP, - index.AsRegister<Register>()); - } - codegen_->MaybeRecordImplicitNullCheck(instruction); - DCHECK(!needs_write_barrier); - DCHECK(!may_need_runtime_call_for_type_check); - break; - } - - DCHECK(needs_write_barrier); - Location temp1_loc = locations->GetTemp(0); - Register temp1 = temp1_loc.AsRegister<Register>(); - Location temp2_loc = locations->GetTemp(1); - Register temp2 = temp2_loc.AsRegister<Register>(); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - Label done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - SlowPathCodeARM* slow_path = nullptr; - - if (may_need_runtime_call_for_type_check) { - slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); - codegen_->AddSlowPath(slow_path); - if (instruction->GetValueCanBeNull()) { - Label non_zero; - __ CompareAndBranchIfNonZero(value, &non_zero); - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreToOffset(kStoreWord, value, array, offset); - } else { - DCHECK(index.IsRegister()) << index; - __ add(IP, array, ShifterOperand(data_offset)); - codegen_->StoreToShiftedRegOffset(value_type, - value_loc, - IP, - index.AsRegister<Register>()); - } - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ b(final_label); - __ Bind(&non_zero); - } - - // Note that when read barriers are enabled, the type checks - // are performed without read barriers. This is fine, even in - // the case where a class object is in the from-space after - // the flip, as a comparison involving such a type would not - // produce a false positive; it may of course produce a false - // negative, in which case we would take the ArraySet slow - // path. - - // /* HeapReference<Class> */ temp1 = array->klass_ - __ LoadFromOffset(kLoadWord, temp1, array, class_offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ MaybeUnpoisonHeapReference(temp1); - - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); - // /* HeapReference<Class> */ temp2 = value->klass_ - __ LoadFromOffset(kLoadWord, temp2, value, class_offset); - // If heap poisoning is enabled, no need to unpoison `temp1` - // nor `temp2`, as we are comparing two poisoned references. - __ cmp(temp1, ShifterOperand(temp2)); - - if (instruction->StaticTypeOfArrayIsObjectArray()) { - Label do_put; - __ b(&do_put, EQ); - // If heap poisoning is enabled, the `temp1` reference has - // not been unpoisoned yet; unpoison it now. - __ MaybeUnpoisonHeapReference(temp1); - - // /* HeapReference<Class> */ temp1 = temp1->super_class_ - __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); - // If heap poisoning is enabled, no need to unpoison - // `temp1`, as we are comparing against null below. - __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); - __ Bind(&do_put); - } else { - __ b(slow_path->GetEntryLabel(), NE); - } - } - - Register source = value; - if (kPoisonHeapReferences) { - // Note that in the case where `value` is a null reference, - // we do not enter this block, as a null reference does not - // need poisoning. - DCHECK_EQ(value_type, Primitive::kPrimNot); - __ Mov(temp1, value); - __ PoisonHeapReference(temp1); - source = temp1; - } - - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreToOffset(kStoreWord, source, array, offset); - } else { - DCHECK(index.IsRegister()) << index; - - __ add(IP, array, ShifterOperand(data_offset)); - codegen_->StoreToShiftedRegOffset(value_type, - Location::RegisterLocation(source), - IP, - index.AsRegister<Register>()); - } - - if (!may_need_runtime_call_for_type_check) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - - codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); - - if (done.IsLinked()) { - __ Bind(&done); - } - - if (slow_path != nullptr) { - __ Bind(slow_path->GetExitLabel()); - } - - break; - } - - case Primitive::kPrimLong: { - Location value = locations->InAt(2); - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); - } else { - __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); - } - break; - } - - case Primitive::kPrimFloat: { - Location value = locations->InAt(2); - DCHECK(value.IsFpuRegister()); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); - } else { - __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); - __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); - } - break; - } - - case Primitive::kPrimDouble: { - Location value = locations->InAt(2); - DCHECK(value.IsFpuRegisterPair()); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); - } else { - __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); - } - - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << value_type; - UNREACHABLE(); - } - - // Objects are handled in the switch. - if (value_type != Primitive::kPrimNot) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } -} - -void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { - LocationSummary* locations = instruction->GetLocations(); - uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); - Register obj = locations->InAt(0).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - __ LoadFromOffset(kLoadWord, out, obj, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - // Mask out compression flag from String's array length. - if (mirror::kUseStringCompression && instruction->IsStringLength()) { - __ Lsr(out, out, 1u); - } -} - -void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - if (second.IsRegister()) { - __ add(out.AsRegister<Register>(), - first.AsRegister<Register>(), - ShifterOperand(second.AsRegister<Register>())); - } else { - __ AddConstant(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.GetConstant()->AsIntConstant()->GetValue()); - } -} - -void LocationsBuilderARM::VisitIntermediateAddressIndex(HIntermediateAddressIndex* instruction) { - LOG(FATAL) << "Unreachable " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitIntermediateAddressIndex( - HIntermediateAddressIndex* instruction) { - LOG(FATAL) << "Unreachable " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { - RegisterSet caller_saves = RegisterSet::Empty(); - InvokeRuntimeCallingConvention calling_convention; - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); - - HInstruction* index = instruction->InputAt(0); - HInstruction* length = instruction->InputAt(1); - // If both index and length are constants we can statically check the bounds. But if at least one - // of them is not encodable ArmEncodableConstantOrRegister will create - // Location::RequiresRegister() which is not desired to happen. Instead we create constant - // locations. - bool both_const = index->IsConstant() && length->IsConstant(); - locations->SetInAt(0, both_const - ? Location::ConstantLocation(index->AsConstant()) - : ArmEncodableConstantOrRegister(index, CMP)); - locations->SetInAt(1, both_const - ? Location::ConstantLocation(length->AsConstant()) - : ArmEncodableConstantOrRegister(length, CMP)); -} - -void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location index_loc = locations->InAt(0); - Location length_loc = locations->InAt(1); - - if (length_loc.IsConstant()) { - int32_t length = helpers::Int32ConstantFrom(length_loc); - if (index_loc.IsConstant()) { - // BCE will remove the bounds check if we are guaranteed to pass. - int32_t index = helpers::Int32ConstantFrom(index_loc); - if (index < 0 || index >= length) { - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel()); - } else { - // Some optimization after BCE may have generated this, and we should not - // generate a bounds check if it is a valid range. - } - return; - } - - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); - __ cmp(index_loc.AsRegister<Register>(), ShifterOperand(length)); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), HS); - } else { - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); - if (index_loc.IsConstant()) { - int32_t index = helpers::Int32ConstantFrom(index_loc); - __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index)); - } else { - __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index_loc.AsRegister<Register>())); - } - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), LS); - } -} - -void CodeGeneratorARM::MarkGCCard(Register temp, - Register card, - Register object, - Register value, - bool can_be_null) { - Label is_null; - if (can_be_null) { - __ CompareAndBranchIfZero(value, &is_null); - } - __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); - __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); - __ strb(card, Address(card, temp)); - if (can_be_null) { - __ Bind(&is_null); - } -} - -void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { - LOG(FATAL) << "Unreachable"; -} - -void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { - codegen_->GetMoveResolver()->EmitNativeCode(instruction); -} - -void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. -} - -void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { - HBasicBlock* block = instruction->GetBlock(); - if (block->GetLoopInformation() != nullptr) { - DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); - // The back edge will generate the suspend check. - return; - } - if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { - // The goto will generate the suspend check. - return; - } - GenerateSuspendCheck(instruction, nullptr); -} - -void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, - HBasicBlock* successor) { - SuspendCheckSlowPathARM* slow_path = - down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); - if (slow_path == nullptr) { - slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); - instruction->SetSlowPath(slow_path); - codegen_->AddSlowPath(slow_path); - if (successor != nullptr) { - DCHECK(successor->IsLoopHeader()); - codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); - } - } else { - DCHECK_EQ(slow_path->GetSuccessor(), successor); - } - - __ LoadFromOffset( - kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); - if (successor == nullptr) { - __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); - __ Bind(slow_path->GetReturnLabel()); - } else { - __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); - __ b(slow_path->GetEntryLabel()); - } -} - -ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { - return codegen_->GetAssembler(); -} - -void ParallelMoveResolverARM::EmitMove(size_t index) { - MoveOperands* move = moves_[index]; - Location source = move->GetSource(); - Location destination = move->GetDestination(); - - if (source.IsRegister()) { - if (destination.IsRegister()) { - __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); - } else if (destination.IsFpuRegister()) { - __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); - } else { - DCHECK(destination.IsStackSlot()); - __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), - SP, destination.GetStackIndex()); - } - } else if (source.IsStackSlot()) { - if (destination.IsRegister()) { - __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), - SP, source.GetStackIndex()); - } else if (destination.IsFpuRegister()) { - __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); - } else { - DCHECK(destination.IsStackSlot()); - __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } else if (source.IsFpuRegister()) { - if (destination.IsRegister()) { - __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); - } else if (destination.IsFpuRegister()) { - __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); - } else { - DCHECK(destination.IsStackSlot()); - __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); - } - } else if (source.IsDoubleStackSlot()) { - if (destination.IsDoubleStackSlot()) { - __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); - __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); - } else if (destination.IsRegisterPair()) { - DCHECK(ExpectedPairLayout(destination)); - __ LoadFromOffset( - kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); - } else { - DCHECK(destination.IsFpuRegisterPair()) << destination; - __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - SP, - source.GetStackIndex()); - } - } else if (source.IsRegisterPair()) { - if (destination.IsRegisterPair()) { - __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); - __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); - } else if (destination.IsFpuRegisterPair()) { - __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - source.AsRegisterPairLow<Register>(), - source.AsRegisterPairHigh<Register>()); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - DCHECK(ExpectedPairLayout(source)); - __ StoreToOffset( - kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); - } - } else if (source.IsFpuRegisterPair()) { - if (destination.IsRegisterPair()) { - __ vmovrrd(destination.AsRegisterPairLow<Register>(), - destination.AsRegisterPairHigh<Register>(), - FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); - } else if (destination.IsFpuRegisterPair()) { - __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), - SP, - destination.GetStackIndex()); - } - } else { - DCHECK(source.IsConstant()) << source; - HConstant* constant = source.GetConstant(); - if (constant->IsIntConstant() || constant->IsNullConstant()) { - int32_t value = CodeGenerator::GetInt32ValueOf(constant); - if (destination.IsRegister()) { - __ LoadImmediate(destination.AsRegister<Register>(), value); - } else { - DCHECK(destination.IsStackSlot()); - __ LoadImmediate(IP, value); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } else if (constant->IsLongConstant()) { - int64_t value = constant->AsLongConstant()->GetValue(); - if (destination.IsRegisterPair()) { - __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); - __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - __ LoadImmediate(IP, Low32Bits(value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - __ LoadImmediate(IP, High32Bits(value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); - } - } else if (constant->IsDoubleConstant()) { - double value = constant->AsDoubleConstant()->GetValue(); - if (destination.IsFpuRegisterPair()) { - __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - uint64_t int_value = bit_cast<uint64_t, double>(value); - __ LoadImmediate(IP, Low32Bits(int_value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - __ LoadImmediate(IP, High32Bits(int_value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); - } - } else { - DCHECK(constant->IsFloatConstant()) << constant->DebugName(); - float value = constant->AsFloatConstant()->GetValue(); - if (destination.IsFpuRegister()) { - __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); - } else { - DCHECK(destination.IsStackSlot()); - __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } - } -} - -void ParallelMoveResolverARM::Exchange(Register reg, int mem) { - __ Mov(IP, reg); - __ LoadFromOffset(kLoadWord, reg, SP, mem); - __ StoreToOffset(kStoreWord, IP, SP, mem); -} - -void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { - ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); - int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; - __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), - SP, mem1 + stack_offset); - __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); - __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), - SP, mem2 + stack_offset); - __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); -} - -void ParallelMoveResolverARM::EmitSwap(size_t index) { - MoveOperands* move = moves_[index]; - Location source = move->GetSource(); - Location destination = move->GetDestination(); - - if (source.IsRegister() && destination.IsRegister()) { - DCHECK_NE(source.AsRegister<Register>(), IP); - DCHECK_NE(destination.AsRegister<Register>(), IP); - __ Mov(IP, source.AsRegister<Register>()); - __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); - __ Mov(destination.AsRegister<Register>(), IP); - } else if (source.IsRegister() && destination.IsStackSlot()) { - Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); - } else if (source.IsStackSlot() && destination.IsRegister()) { - Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); - } else if (source.IsStackSlot() && destination.IsStackSlot()) { - Exchange(source.GetStackIndex(), destination.GetStackIndex()); - } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { - __ vmovrs(IP, source.AsFpuRegister<SRegister>()); - __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); - __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); - } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { - __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); - __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); - __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); - __ vmovrrd(destination.AsRegisterPairLow<Register>(), - destination.AsRegisterPairHigh<Register>(), - DTMP); - } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { - Register low_reg = source.IsRegisterPair() - ? source.AsRegisterPairLow<Register>() - : destination.AsRegisterPairLow<Register>(); - int mem = source.IsRegisterPair() - ? destination.GetStackIndex() - : source.GetStackIndex(); - DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); - __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); - __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); - __ StoreDToOffset(DTMP, SP, mem); - } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { - DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); - DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); - __ vmovd(DTMP, first); - __ vmovd(first, second); - __ vmovd(second, DTMP); - } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { - DRegister reg = source.IsFpuRegisterPair() - ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) - : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); - int mem = source.IsFpuRegisterPair() - ? destination.GetStackIndex() - : source.GetStackIndex(); - __ vmovd(DTMP, reg); - __ LoadDFromOffset(reg, SP, mem); - __ StoreDToOffset(DTMP, SP, mem); - } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { - SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() - : destination.AsFpuRegister<SRegister>(); - int mem = source.IsFpuRegister() - ? destination.GetStackIndex() - : source.GetStackIndex(); - - __ vmovrs(IP, reg); - __ LoadSFromOffset(reg, SP, mem); - __ StoreToOffset(kStoreWord, IP, SP, mem); - } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { - Exchange(source.GetStackIndex(), destination.GetStackIndex()); - Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); - } else { - LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; - } -} - -void ParallelMoveResolverARM::SpillScratch(int reg) { - __ Push(static_cast<Register>(reg)); -} - -void ParallelMoveResolverARM::RestoreScratch(int reg) { - __ Pop(static_cast<Register>(reg)); -} - -HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( - HLoadClass::LoadKind desired_class_load_kind) { - switch (desired_class_load_kind) { - case HLoadClass::LoadKind::kInvalid: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - case HLoadClass::LoadKind::kReferrersClass: - break; - case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBssEntry: - DCHECK(!Runtime::Current()->UseJitCompilation()); - break; - case HLoadClass::LoadKind::kJitTableAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); - break; - case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kRuntimeCall: - break; - } - return desired_class_load_kind; -} - -void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { - HLoadClass::LoadKind load_kind = cls->GetLoadKind(); - if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { - InvokeRuntimeCallingConvention calling_convention; - CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( - cls, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Location::RegisterLocation(R0)); - DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); - return; - } - DCHECK(!cls->NeedsAccessCheck()); - - const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); - LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall; - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); - if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - - if (load_kind == HLoadClass::LoadKind::kReferrersClass) { - locations->SetInAt(0, Location::RequiresRegister()); - } - locations->SetOut(Location::RequiresRegister()); - if (load_kind == HLoadClass::LoadKind::kBssEntry) { - if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the type resolution or initialization and marking to save everything we need. - // Note that IP may be clobbered by saving/restoring the live register (only one thanks - // to the custom calling convention) or by marking, so we request a different temp. - locations->AddTemp(Location::RequiresRegister()); - RegisterSet caller_saves = RegisterSet::Empty(); - InvokeRuntimeCallingConvention calling_convention; - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() - // that the the kPrimNot result register is the same as the first argument register. - locations->SetCustomSlowPathCallerSaves(caller_saves); - } else { - // For non-Baker read barrier we have a temp-clobbering call. - } - } - if (kUseBakerReadBarrier && kBakerReadBarrierLinkTimeThunksEnableForGcRoots) { - if (load_kind == HLoadClass::LoadKind::kBssEntry || - (load_kind == HLoadClass::LoadKind::kReferrersClass && - !Runtime::Current()->UseJitCompilation())) { - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } - } -} - -// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not -// move. -void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { - HLoadClass::LoadKind load_kind = cls->GetLoadKind(); - if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { - codegen_->GenerateLoadClassRuntimeCall(cls); - return; - } - DCHECK(!cls->NeedsAccessCheck()); - - LocationSummary* locations = cls->GetLocations(); - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); - - const ReadBarrierOption read_barrier_option = cls->IsInBootImage() - ? kWithoutReadBarrier - : kCompilerReadBarrierOption; - bool generate_null_check = false; - switch (load_kind) { - case HLoadClass::LoadKind::kReferrersClass: { - DCHECK(!cls->CanCallRuntime()); - DCHECK(!cls->MustGenerateClinitCheck()); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - Register current_method = locations->InAt(0).AsRegister<Register>(); - GenerateGcRootFieldLoad(cls, - out_loc, - current_method, - ArtMethod::DeclaringClassOffset().Int32Value(), - read_barrier_option); - break; - } - case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { - DCHECK(codegen_->GetCompilerOptions().IsBootImage()); - DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(out, out, ShifterOperand(PC)); - break; - } - case HLoadClass::LoadKind::kBootImageAddress: { - DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); - uint32_t address = dchecked_integral_cast<uint32_t>( - reinterpret_cast<uintptr_t>(cls->GetClass().Get())); - DCHECK_NE(address, 0u); - __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); - break; - } - case HLoadClass::LoadKind::kBssEntry: { - Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) - ? locations->GetTemp(0).AsRegister<Register>() - : out; - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); - generate_null_check = true; - break; - } - case HLoadClass::LoadKind::kJitTableAddress: { - __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), - cls->GetTypeIndex(), - cls->GetClass())); - // /* GcRoot<mirror::Class> */ out = *out - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); - break; - } - case HLoadClass::LoadKind::kRuntimeCall: - case HLoadClass::LoadKind::kInvalid: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } - - if (generate_null_check || cls->MustGenerateClinitCheck()) { - DCHECK(cls->CanCallRuntime()); - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( - cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); - codegen_->AddSlowPath(slow_path); - if (generate_null_check) { - __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); - } - if (cls->MustGenerateClinitCheck()) { - GenerateClassInitializationCheck(slow_path, out); - } else { - __ Bind(slow_path->GetExitLabel()); - } - } -} - -void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); - locations->SetInAt(0, Location::RequiresRegister()); - if (check->HasUses()) { - locations->SetOut(Location::SameAsFirstInput()); - } -} - -void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { - // We assume the class is not null. - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( - check->GetLoadClass(), check, check->GetDexPc(), true); - codegen_->AddSlowPath(slow_path); - GenerateClassInitializationCheck(slow_path, - check->GetLocations()->InAt(0).AsRegister<Register>()); -} - -void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( - SlowPathCodeARM* slow_path, Register class_reg) { - __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); - __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); - __ b(slow_path->GetEntryLabel(), LT); - // Even if the initialized flag is set, we may be in a situation where caches are not synced - // properly. Therefore, we do a memory fence. - __ dmb(ISH); - __ Bind(slow_path->GetExitLabel()); -} - -HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( - HLoadString::LoadKind desired_string_load_kind) { - switch (desired_string_load_kind) { - case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBssEntry: - DCHECK(!Runtime::Current()->UseJitCompilation()); - break; - case HLoadString::LoadKind::kJitTableAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); - break; - case HLoadString::LoadKind::kBootImageAddress: - case HLoadString::LoadKind::kRuntimeCall: - break; - } - return desired_string_load_kind; -} - -void LocationsBuilderARM::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - HLoadString::LoadKind load_kind = load->GetLoadKind(); - if (load_kind == HLoadString::LoadKind::kRuntimeCall) { - locations->SetOut(Location::RegisterLocation(R0)); - } else { - locations->SetOut(Location::RequiresRegister()); - if (load_kind == HLoadString::LoadKind::kBssEntry) { - if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and marking to save everything we need, including temps. - // Note that IP may be clobbered by saving/restoring the live register (only one thanks - // to the custom calling convention) or by marking, so we request a different temp. - locations->AddTemp(Location::RequiresRegister()); - RegisterSet caller_saves = RegisterSet::Empty(); - InvokeRuntimeCallingConvention calling_convention; - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() - // that the the kPrimNot result register is the same as the first argument register. - locations->SetCustomSlowPathCallerSaves(caller_saves); - if (kUseBakerReadBarrier && kBakerReadBarrierLinkTimeThunksEnableForGcRoots) { - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } - } else { - // For non-Baker read barrier we have a temp-clobbering call. - } - } - } -} - -// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not -// move. -void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { - LocationSummary* locations = load->GetLocations(); - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); - HLoadString::LoadKind load_kind = load->GetLoadKind(); - - switch (load_kind) { - case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { - DCHECK(codegen_->GetCompilerOptions().IsBootImage()); - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(out, out, ShifterOperand(PC)); - return; // No dex cache slow path. - } - case HLoadString::LoadKind::kBootImageAddress: { - uint32_t address = dchecked_integral_cast<uint32_t>( - reinterpret_cast<uintptr_t>(load->GetString().Get())); - DCHECK_NE(address, 0u); - __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); - return; // No dex cache slow path. - } - case HLoadString::LoadKind::kBssEntry: { - DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) - ? locations->GetTemp(0).AsRegister<Register>() - : out; - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); - SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); - codegen_->AddSlowPath(slow_path); - __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); - return; - } - case HLoadString::LoadKind::kJitTableAddress: { - __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), - load->GetStringIndex(), - load->GetString())); - // /* GcRoot<mirror::String> */ out = *out - GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); - return; - } - default: - break; - } - - // TODO: Consider re-adding the compiler code to do string dex cache lookup again. - DCHECK(load_kind == HLoadString::LoadKind::kRuntimeCall); - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), out); - __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); - codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); - CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); -} - -static int32_t GetExceptionTlsOffset() { - return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); -} - -void LocationsBuilderARM::VisitLoadException(HLoadException* load) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { - Register out = load->GetLocations()->Out().AsRegister<Register>(); - __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); -} - -void LocationsBuilderARM::VisitClearException(HClearException* clear) { - new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); -} - -void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { - __ LoadImmediate(IP, 0); - __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); -} - -void LocationsBuilderARM::VisitThrow(HThrow* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); -} - -void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { - codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); -} - -// Temp is used for read barrier. -static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { - if (kEmitCompilerReadBarrier && - (kUseBakerReadBarrier || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck)) { - return 1; - } - return 0; -} - -// Interface case has 3 temps, one for holding the number of interfaces, one for the current -// interface pointer, one for loading the current interface. -// The other checks have one temp for loading the object's class. -static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { - if (type_check_kind == TypeCheckKind::kInterfaceCheck) { - return 3; - } - return 1 + NumberOfInstanceOfTemps(type_check_kind); -} - -void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { - LocationSummary::CallKind call_kind = LocationSummary::kNoCall; - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - bool baker_read_barrier_slow_path = false; - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: - case TypeCheckKind::kAbstractClassCheck: - case TypeCheckKind::kClassHierarchyCheck: - case TypeCheckKind::kArrayObjectCheck: - call_kind = - kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; - baker_read_barrier_slow_path = kUseBakerReadBarrier; - break; - case TypeCheckKind::kArrayCheck: - case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - call_kind = LocationSummary::kCallOnSlowPath; - break; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); - if (baker_read_barrier_slow_path) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - // The "out" register is used as a temporary, so it overlaps with the inputs. - // Note that TypeCheckSlowPathARM uses this register too. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - codegen_->MaybeAddBakerCcEntrypointTempForFields(locations); - } -} - -void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - LocationSummary* locations = instruction->GetLocations(); - Location obj_loc = locations->InAt(0); - Register obj = obj_loc.AsRegister<Register>(); - Register cls = locations->InAt(1).AsRegister<Register>(); - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); - const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); - DCHECK_LE(num_temps, 1u); - Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - Label done; - Label* const final_label = codegen_->GetFinalLabel(instruction, &done); - SlowPathCodeARM* slow_path = nullptr; - - // Return 0 if `obj` is null. - // avoid null check if we know obj is not null. - if (instruction->MustDoNullCheck()) { - DCHECK_NE(out, obj); - __ LoadImmediate(out, 0); - __ CompareAndBranchIfZero(obj, final_label); - } - - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // Classes must be equal for the instanceof to succeed. - __ cmp(out, ShifterOperand(cls)); - // We speculatively set the result to false without changing the condition - // flags, which allows us to avoid some branching later. - __ mov(out, ShifterOperand(0), AL, kCcKeep); - - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the output is in a low register, so that a 16-bit MOV - // encoding can be used. - if (ArmAssembler::IsLowRegister(out)) { - __ it(EQ); - __ mov(out, ShifterOperand(1), EQ); - } else { - __ b(final_label, NE); - __ LoadImmediate(out, 1); - } - - break; - } - - case TypeCheckKind::kAbstractClassCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // If the class is abstract, we eagerly fetch the super class of the - // object to avoid doing a comparison we know will fail. - Label loop; - __ Bind(&loop); - // /* HeapReference<Class> */ out = out->super_class_ - GenerateReferenceLoadOneRegister(instruction, - out_loc, - super_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // If `out` is null, we use it for the result, and jump to the final label. - __ CompareAndBranchIfZero(out, final_label); - __ cmp(out, ShifterOperand(cls)); - __ b(&loop, NE); - __ LoadImmediate(out, 1); - break; - } - - case TypeCheckKind::kClassHierarchyCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // Walk over the class hierarchy to find a match. - Label loop, success; - __ Bind(&loop); - __ cmp(out, ShifterOperand(cls)); - __ b(&success, EQ); - // /* HeapReference<Class> */ out = out->super_class_ - GenerateReferenceLoadOneRegister(instruction, - out_loc, - super_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // This is essentially a null check, but it sets the condition flags to the - // proper value for the code that follows the loop, i.e. not `EQ`. - __ cmp(out, ShifterOperand(1)); - __ b(&loop, HS); - - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the output is in a low register, so that a 16-bit MOV - // encoding can be used. - if (ArmAssembler::IsLowRegister(out)) { - // If `out` is null, we use it for the result, and the condition flags - // have already been set to `NE`, so the IT block that comes afterwards - // (and which handles the successful case) turns into a NOP (instead of - // overwriting `out`). - __ Bind(&success); - // There is only one branch to the `success` label (which is bound to this - // IT block), and it has the same condition, `EQ`, so in that case the MOV - // is executed. - __ it(EQ); - __ mov(out, ShifterOperand(1), EQ); - } else { - // If `out` is null, we use it for the result, and jump to the final label. - __ b(final_label); - __ Bind(&success); - __ LoadImmediate(out, 1); - } - - break; - } - - case TypeCheckKind::kArrayObjectCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // Do an exact check. - Label exact_check; - __ cmp(out, ShifterOperand(cls)); - __ b(&exact_check, EQ); - // Otherwise, we need to check that the object's class is a non-primitive array. - // /* HeapReference<Class> */ out = out->component_type_ - GenerateReferenceLoadOneRegister(instruction, - out_loc, - component_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // If `out` is null, we use it for the result, and jump to the final label. - __ CompareAndBranchIfZero(out, final_label); - __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ cmp(out, ShifterOperand(0)); - // We speculatively set the result to false without changing the condition - // flags, which allows us to avoid some branching later. - __ mov(out, ShifterOperand(0), AL, kCcKeep); - - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the output is in a low register, so that a 16-bit MOV - // encoding can be used. - if (ArmAssembler::IsLowRegister(out)) { - __ Bind(&exact_check); - __ it(EQ); - __ mov(out, ShifterOperand(1), EQ); - } else { - __ b(final_label, NE); - __ Bind(&exact_check); - __ LoadImmediate(out, 1); - } - - break; - } - - case TypeCheckKind::kArrayCheck: { - // No read barrier since the slow path will retry upon failure. - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kWithoutReadBarrier); - __ cmp(out, ShifterOperand(cls)); - DCHECK(locations->OnlyCallsOnSlowPath()); - slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, - /* is_fatal */ false); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), NE); - __ LoadImmediate(out, 1); - break; - } - - case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: { - // Note that we indeed only call on slow path, but we always go - // into the slow path for the unresolved and interface check - // cases. - // - // We cannot directly call the InstanceofNonTrivial runtime - // entry point without resorting to a type checking slow path - // here (i.e. by calling InvokeRuntime directly), as it would - // require to assign fixed registers for the inputs of this - // HInstanceOf instruction (following the runtime calling - // convention), which might be cluttered by the potential first - // read barrier emission at the beginning of this method. - // - // TODO: Introduce a new runtime entry point taking the object - // to test (instead of its class) as argument, and let it deal - // with the read barrier issues. This will let us refactor this - // case of the `switch` code as it was previously (with a direct - // call to the runtime not using a type checking slow path). - // This should also be beneficial for the other cases above. - DCHECK(locations->OnlyCallsOnSlowPath()); - slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, - /* is_fatal */ false); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel()); - break; - } - } - - if (done.IsLinked()) { - __ Bind(&done); - } - - if (slow_path != nullptr) { - __ Bind(slow_path->GetExitLabel()); - } -} - -void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { - LocationSummary::CallKind call_kind = LocationSummary::kNoCall; - bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); - - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: - case TypeCheckKind::kAbstractClassCheck: - case TypeCheckKind::kClassHierarchyCheck: - case TypeCheckKind::kArrayObjectCheck: - call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. - break; - case TypeCheckKind::kArrayCheck: - case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - call_kind = LocationSummary::kCallOnSlowPath; - break; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); -} - -void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - LocationSummary* locations = instruction->GetLocations(); - Location obj_loc = locations->InAt(0); - Register obj = obj_loc.AsRegister<Register>(); - Register cls = locations->InAt(1).AsRegister<Register>(); - Location temp_loc = locations->GetTemp(0); - Register temp = temp_loc.AsRegister<Register>(); - const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); - DCHECK_LE(num_temps, 3u); - Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); - Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); - const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); - const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); - const uint32_t object_array_data_offset = - mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); - - // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases - // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding - // read barriers is done for performance and code size reasons. - bool is_type_check_slow_path_fatal = false; - if (!kEmitCompilerReadBarrier) { - is_type_check_slow_path_fatal = - (type_check_kind == TypeCheckKind::kExactCheck || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck) && - !instruction->CanThrowIntoCatchBlock(); - } - SlowPathCodeARM* type_check_slow_path = - new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, - is_type_check_slow_path_fatal); - codegen_->AddSlowPath(type_check_slow_path); - - Label done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - // Avoid null check if we know obj is not null. - if (instruction->MustDoNullCheck()) { - __ CompareAndBranchIfZero(obj, final_label); - } - - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: - case TypeCheckKind::kArrayCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - __ cmp(temp, ShifterOperand(cls)); - // Jump to slow path for throwing the exception or doing a - // more involved array check. - __ b(type_check_slow_path->GetEntryLabel(), NE); - break; - } - - case TypeCheckKind::kAbstractClassCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // If the class is abstract, we eagerly fetch the super class of the - // object to avoid doing a comparison we know will fail. - Label loop; - __ Bind(&loop); - // /* HeapReference<Class> */ temp = temp->super_class_ - GenerateReferenceLoadOneRegister(instruction, - temp_loc, - super_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // If the class reference currently in `temp` is null, jump to the slow path to throw the - // exception. - __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); - - // Otherwise, compare the classes. - __ cmp(temp, ShifterOperand(cls)); - __ b(&loop, NE); - break; - } - - case TypeCheckKind::kClassHierarchyCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // Walk over the class hierarchy to find a match. - Label loop; - __ Bind(&loop); - __ cmp(temp, ShifterOperand(cls)); - __ b(final_label, EQ); - - // /* HeapReference<Class> */ temp = temp->super_class_ - GenerateReferenceLoadOneRegister(instruction, - temp_loc, - super_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // If the class reference currently in `temp` is null, jump to the slow path to throw the - // exception. - __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); - // Otherwise, jump to the beginning of the loop. - __ b(&loop); - break; - } - - case TypeCheckKind::kArrayObjectCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // Do an exact check. - __ cmp(temp, ShifterOperand(cls)); - __ b(final_label, EQ); - - // Otherwise, we need to check that the object's class is a non-primitive array. - // /* HeapReference<Class> */ temp = temp->component_type_ - GenerateReferenceLoadOneRegister(instruction, - temp_loc, - component_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - // If the component type is null, jump to the slow path to throw the exception. - __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); - // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` - // to further check that this component type is not a primitive type. - __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); - __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); - break; - } - - case TypeCheckKind::kUnresolvedCheck: - // We always go into the type check slow path for the unresolved check case. - // We cannot directly call the CheckCast runtime entry point - // without resorting to a type checking slow path here (i.e. by - // calling InvokeRuntime directly), as it would require to - // assign fixed registers for the inputs of this HInstanceOf - // instruction (following the runtime calling convention), which - // might be cluttered by the potential first read barrier - // emission at the beginning of this method. - - __ b(type_check_slow_path->GetEntryLabel()); - break; - - case TypeCheckKind::kInterfaceCheck: { - // Avoid read barriers to improve performance of the fast path. We can not get false - // positives by doing this. - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // /* HeapReference<Class> */ temp = temp->iftable_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - temp_loc, - iftable_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - // Iftable is never null. - __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); - // Loop through the iftable and check if any class matches. - Label start_loop; - __ Bind(&start_loop); - __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), - type_check_slow_path->GetEntryLabel()); - __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); - __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); - // Go to next interface. - __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); - __ sub(maybe_temp2_loc.AsRegister<Register>(), - maybe_temp2_loc.AsRegister<Register>(), - ShifterOperand(2)); - // Compare the classes and continue the loop if they do not match. - __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); - __ b(&start_loop, NE); - break; - } - } - - if (done.IsLinked()) { - __ Bind(&done); - } - - __ Bind(type_check_slow_path->GetExitLabel()); -} - -void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); -} - -void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { - codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, - instruction, - instruction->GetDexPc()); - if (instruction->IsEnter()) { - CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); - } else { - CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); - } -} - -void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } -void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } -void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } - -void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt - || instruction->GetResultType() == Primitive::kPrimLong); - // Note: GVN reorders commutative operations to have the constant on the right hand side. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { - HandleBitwiseOperation(instruction); -} - -void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { - HandleBitwiseOperation(instruction); -} - -void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { - HandleBitwiseOperation(instruction); -} - - -void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt - || instruction->GetResultType() == Primitive::kPrimLong); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - Location out = locations->Out(); - - if (instruction->GetResultType() == Primitive::kPrimInt) { - Register first_reg = first.AsRegister<Register>(); - ShifterOperand second_reg(second.AsRegister<Register>()); - Register out_reg = out.AsRegister<Register>(); - - switch (instruction->GetOpKind()) { - case HInstruction::kAnd: - __ bic(out_reg, first_reg, second_reg); - break; - case HInstruction::kOr: - __ orn(out_reg, first_reg, second_reg); - break; - // There is no EON on arm. - case HInstruction::kXor: - default: - LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); - UNREACHABLE(); - } - return; - - } else { - DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - ShifterOperand second_low(second.AsRegisterPairLow<Register>()); - ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - - switch (instruction->GetOpKind()) { - case HInstruction::kAnd: - __ bic(out_low, first_low, second_low); - __ bic(out_high, first_high, second_high); - break; - case HInstruction::kOr: - __ orn(out_low, first_low, second_low); - __ orn(out_high, first_high, second_high); - break; - // There is no EON on arm. - case HInstruction::kXor: - default: - LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); - UNREACHABLE(); - } - } -} - -void LocationsBuilderARM::VisitDataProcWithShifterOp( - HDataProcWithShifterOp* instruction) { - DCHECK(instruction->GetType() == Primitive::kPrimInt || - instruction->GetType() == Primitive::kPrimLong); - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - const bool overlap = instruction->GetType() == Primitive::kPrimLong && - HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), - overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitDataProcWithShifterOp( - HDataProcWithShifterOp* instruction) { - const LocationSummary* const locations = instruction->GetLocations(); - const HInstruction::InstructionKind kind = instruction->GetInstrKind(); - const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - const Location out = locations->Out(); - - if (instruction->GetType() == Primitive::kPrimInt) { - DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); - - const Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong - ? right.AsRegisterPairLow<Register>() - : right.AsRegister<Register>(); - - GenerateDataProcInstruction(kind, - out.AsRegister<Register>(), - left.AsRegister<Register>(), - ShifterOperand(second, - ShiftFromOpKind(op_kind), - instruction->GetShiftAmount()), - codegen_); - } else { - DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); - - if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { - const Register second = right.AsRegister<Register>(); - - DCHECK_NE(out.AsRegisterPairLow<Register>(), second); - GenerateDataProc(kind, - out, - left, - ShifterOperand(second), - ShifterOperand(second, ASR, 31), - codegen_); - } else { - GenerateLongDataProc(instruction, codegen_); - } - } -} - -void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { - // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). - if (value == 0xffffffffu) { - if (out != first) { - __ mov(out, ShifterOperand(first)); - } - return; - } - if (value == 0u) { - __ mov(out, ShifterOperand(0)); - return; - } - ShifterOperand so; - if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { - __ and_(out, first, so); - } else if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)) { - __ bic(out, first, ShifterOperand(~value)); - } else { - DCHECK(IsPowerOfTwo(value + 1)); - __ ubfx(out, first, 0, WhichPowerOf2(value + 1)); - } -} - -void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { - // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). - if (value == 0u) { - if (out != first) { - __ mov(out, ShifterOperand(first)); - } - return; - } - if (value == 0xffffffffu) { - __ mvn(out, ShifterOperand(0)); - return; - } - ShifterOperand so; - if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { - __ orr(out, first, so); - } else { - DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); - __ orn(out, first, ShifterOperand(~value)); - } -} - -void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { - // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). - if (value == 0u) { - if (out != first) { - __ mov(out, ShifterOperand(first)); - } - return; - } - __ eor(out, first, ShifterOperand(value)); -} - -void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, - Location first, - uint64_t value) { - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - uint32_t value_low = Low32Bits(value); - uint32_t value_high = High32Bits(value); - if (value_low == 0u) { - if (out_low != first_low) { - __ mov(out_low, ShifterOperand(first_low)); - } - __ AddConstant(out_high, first_high, value_high); - return; - } - __ AddConstantSetFlags(out_low, first_low, value_low); - ShifterOperand so; - if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { - __ adc(out_high, first_high, so); - } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { - __ sbc(out_high, first_high, so); - } else { - LOG(FATAL) << "Unexpected constant " << value_high; - UNREACHABLE(); - } -} - -void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - Location out = locations->Out(); - - if (second.IsConstant()) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); - uint32_t value_low = Low32Bits(value); - if (instruction->GetResultType() == Primitive::kPrimInt) { - Register first_reg = first.AsRegister<Register>(); - Register out_reg = out.AsRegister<Register>(); - if (instruction->IsAnd()) { - GenerateAndConst(out_reg, first_reg, value_low); - } else if (instruction->IsOr()) { - GenerateOrrConst(out_reg, first_reg, value_low); - } else { - DCHECK(instruction->IsXor()); - GenerateEorConst(out_reg, first_reg, value_low); - } - } else { - DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); - uint32_t value_high = High32Bits(value); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - if (instruction->IsAnd()) { - GenerateAndConst(out_low, first_low, value_low); - GenerateAndConst(out_high, first_high, value_high); - } else if (instruction->IsOr()) { - GenerateOrrConst(out_low, first_low, value_low); - GenerateOrrConst(out_high, first_high, value_high); - } else { - DCHECK(instruction->IsXor()); - GenerateEorConst(out_low, first_low, value_low); - GenerateEorConst(out_high, first_high, value_high); - } - } - return; - } - - if (instruction->GetResultType() == Primitive::kPrimInt) { - Register first_reg = first.AsRegister<Register>(); - ShifterOperand second_reg(second.AsRegister<Register>()); - Register out_reg = out.AsRegister<Register>(); - if (instruction->IsAnd()) { - __ and_(out_reg, first_reg, second_reg); - } else if (instruction->IsOr()) { - __ orr(out_reg, first_reg, second_reg); - } else { - DCHECK(instruction->IsXor()); - __ eor(out_reg, first_reg, second_reg); - } - } else { - DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - ShifterOperand second_low(second.AsRegisterPairLow<Register>()); - ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - if (instruction->IsAnd()) { - __ and_(out_low, first_low, second_low); - __ and_(out_high, first_high, second_high); - } else if (instruction->IsOr()) { - __ orr(out_low, first_low, second_low); - __ orr(out_high, first_high, second_high); - } else { - DCHECK(instruction->IsXor()); - __ eor(out_low, first_low, second_low); - __ eor(out_high, first_high, second_high); - } - } -} - -void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( - HInstruction* instruction, - Location out, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option) { - Register out_reg = out.AsRegister<Register>(); - if (read_barrier_option == kWithReadBarrier) { - CHECK(kEmitCompilerReadBarrier); - DCHECK(maybe_temp.IsRegister()) << maybe_temp; - if (kUseBakerReadBarrier) { - // Load with fast path based Baker's read barrier. - // /* HeapReference<Object> */ out = *(out + offset) - codegen_->GenerateFieldLoadWithBakerReadBarrier( - instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); - } else { - // Load with slow path based read barrier. - // Save the value of `out` into `maybe_temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - __ Mov(maybe_temp.AsRegister<Register>(), out_reg); - // /* HeapReference<Object> */ out = *(out + offset) - __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); - codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); - } - } else { - // Plain load with no read barrier. - // /* HeapReference<Object> */ out = *(out + offset) - __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); - __ MaybeUnpoisonHeapReference(out_reg); - } -} - -void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( - HInstruction* instruction, - Location out, - Location obj, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option) { - Register out_reg = out.AsRegister<Register>(); - Register obj_reg = obj.AsRegister<Register>(); - if (read_barrier_option == kWithReadBarrier) { - CHECK(kEmitCompilerReadBarrier); - if (kUseBakerReadBarrier) { - DCHECK(maybe_temp.IsRegister()) << maybe_temp; - // Load with fast path based Baker's read barrier. - // /* HeapReference<Object> */ out = *(obj + offset) - codegen_->GenerateFieldLoadWithBakerReadBarrier( - instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); - } else { - // Load with slow path based read barrier. - // /* HeapReference<Object> */ out = *(obj + offset) - __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); - codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); - } - } else { - // Plain load with no read barrier. - // /* HeapReference<Object> */ out = *(obj + offset) - __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); - __ MaybeUnpoisonHeapReference(out_reg); - } -} - -void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, - Location root, - Register obj, - uint32_t offset, - ReadBarrierOption read_barrier_option) { - Register root_reg = root.AsRegister<Register>(); - if (read_barrier_option == kWithReadBarrier) { - DCHECK(kEmitCompilerReadBarrier); - if (kUseBakerReadBarrier) { - // Fast path implementation of art::ReadBarrier::BarrierForRoot when - // Baker's read barrier are used. - if (kBakerReadBarrierLinkTimeThunksEnableForGcRoots && - !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded GC root or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // We use link-time generated thunks for the slow path. That thunk - // checks the reference and jumps to the entrypoint if needed. - // - // temp = Thread::Current()->pReadBarrierMarkIntrospection - // lr = &return_address; - // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { - // goto gc_root_thunk<root_reg>(lr) - // } - // return_address: - - CheckLastTempIsBakerCcEntrypointRegister(instruction); - bool narrow = CanEmitNarrowLdr(root_reg, obj, offset); - uint32_t custom_data = - linker::Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow); - Label* bne_label = codegen_->NewBakerReadBarrierPatch(custom_data); - - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(IP, 12); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(IP); - __ LoadFromOffset(kLoadWord, kBakerCcEntrypointRegister, TR, entry_point_offset); - - Label return_address; - __ AdrCode(LR, &return_address); - __ CmpConstant(kBakerCcEntrypointRegister, 0); - // Currently the offset is always within range. If that changes, - // we shall have to split the load the same way as for fields. - DCHECK_LT(offset, kReferenceLoadMinFarOffset); - DCHECK(!down_cast<Thumb2Assembler*>(GetAssembler())->IsForced32Bit()); - ScopedForce32Bit maybe_force_32bit(down_cast<Thumb2Assembler*>(GetAssembler()), !narrow); - int old_position = GetAssembler()->GetBuffer()->GetPosition(); - __ LoadFromOffset(kLoadWord, root_reg, obj, offset); - EmitPlaceholderBne(codegen_, bne_label); - __ Bind(&return_address); - DCHECK_EQ(old_position - GetAssembler()->GetBuffer()->GetPosition(), - narrow ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET - : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_OFFSET); - } else { - // Note that we do not actually check the value of - // `GetIsGcMarking()` to decide whether to mark the loaded GC - // root or not. Instead, we load into `temp` the read barrier - // mark entry point corresponding to register `root`. If `temp` - // is null, it means that `GetIsGcMarking()` is false, and vice - // versa. - // - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() - // // Slow path. - // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. - // } - - // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. - Location temp = Location::RegisterLocation(LR); - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( - instruction, root, /* entrypoint */ temp); - codegen_->AddSlowPath(slow_path); - - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); - - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ LoadFromOffset(kLoadWord, root_reg, obj, offset); - static_assert( - sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), - "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " - "have different sizes."); - static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::CompressedReference<mirror::Object> and int32_t " - "have different sizes."); - - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); - } - } else { - // GC root loaded through a slow path for read barriers other - // than Baker's. - // /* GcRoot<mirror::Object>* */ root = obj + offset - __ AddConstant(root_reg, obj, offset); - // /* mirror::Object* */ root = root->Read() - codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); - } - } else { - // Plain GC root load with no read barrier. - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ LoadFromOffset(kLoadWord, root_reg, obj, offset); - // Note that GC roots are not affected by heap poisoning, thus we - // do not have to unpoison `root_reg` here. - } -} - -void CodeGeneratorARM::MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - if (kBakerReadBarrierLinkTimeThunksEnableForFields) { - if (!Runtime::Current()->UseJitCompilation()) { - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } - } -} - -void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location temp, - bool needs_null_check) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - if (kBakerReadBarrierLinkTimeThunksEnableForFields && - !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // We use link-time generated thunks for the slow path. That thunk checks - // the holder and jumps to the entrypoint if needed. If the holder is not - // gray, it creates a fake dependency and returns to the LDR instruction. - // - // temp = Thread::Current()->pReadBarrierMarkIntrospection - // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) - // } - // not_gray_return_address: - // // Original reference load. If the offset is too large to fit - // // into LDR, we use an adjusted base register here. - // HeapReference<mirror::Object> reference = *(obj+offset); - // gray_return_address: - - DCHECK_ALIGNED(offset, sizeof(mirror::HeapReference<mirror::Object>)); - Register ref_reg = ref.AsRegister<Register>(); - bool narrow = CanEmitNarrowLdr(ref_reg, obj, offset); - Register base = obj; - if (offset >= kReferenceLoadMinFarOffset) { - base = temp.AsRegister<Register>(); - DCHECK_NE(base, kBakerCcEntrypointRegister); - static_assert(IsPowerOfTwo(kReferenceLoadMinFarOffset), "Expecting a power of 2."); - __ AddConstant(base, obj, offset & ~(kReferenceLoadMinFarOffset - 1u)); - offset &= (kReferenceLoadMinFarOffset - 1u); - // Use narrow LDR only for small offsets. Generating narrow encoding LDR for the large - // offsets with `(offset & (kReferenceLoadMinFarOffset - 1u)) < 32u` would most likely - // increase the overall code size when taking the generated thunks into account. - DCHECK(!narrow); - } - CheckLastTempIsBakerCcEntrypointRegister(instruction); - uint32_t custom_data = - linker::Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base, obj, narrow); - Label* bne_label = NewBakerReadBarrierPatch(custom_data); - - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(IP, 12); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(IP); - __ LoadFromOffset(kLoadWord, kBakerCcEntrypointRegister, TR, entry_point_offset); - - Label return_address; - __ AdrCode(LR, &return_address); - __ CmpConstant(kBakerCcEntrypointRegister, 0); - EmitPlaceholderBne(this, bne_label); - DCHECK_LT(offset, kReferenceLoadMinFarOffset); - DCHECK(!down_cast<Thumb2Assembler*>(GetAssembler())->IsForced32Bit()); - ScopedForce32Bit maybe_force_32bit(down_cast<Thumb2Assembler*>(GetAssembler()), !narrow); - int old_position = GetAssembler()->GetBuffer()->GetPosition(); - __ LoadFromOffset(kLoadWord, ref_reg, base, offset); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); - __ Bind(&return_address); - DCHECK_EQ(old_position - GetAssembler()->GetBuffer()->GetPosition(), - narrow ? BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET - : BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET); - return; - } - - // /* HeapReference<Object> */ ref = *(obj + offset) - Location no_index = Location::NoLocation(); - ScaleFactor no_scale_factor = TIMES_1; - GenerateReferenceLoadWithBakerReadBarrier( - instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); -} - -void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t data_offset, - Location index, - Location temp, - bool needs_null_check) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - ScaleFactor scale_factor = TIMES_4; - - if (kBakerReadBarrierLinkTimeThunksEnableForArrays && - !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // We use link-time generated thunks for the slow path. That thunk checks - // the holder and jumps to the entrypoint if needed. If the holder is not - // gray, it creates a fake dependency and returns to the LDR instruction. - // - // temp = Thread::Current()->pReadBarrierMarkIntrospection - // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) - // } - // not_gray_return_address: - // // Original reference load. If the offset is too large to fit - // // into LDR, we use an adjusted base register here. - // HeapReference<mirror::Object> reference = data[index]; - // gray_return_address: - - DCHECK(index.IsValid()); - Register index_reg = index.AsRegister<Register>(); - Register ref_reg = ref.AsRegister<Register>(); - Register data_reg = temp.AsRegister<Register>(); - DCHECK_NE(data_reg, kBakerCcEntrypointRegister); - - CheckLastTempIsBakerCcEntrypointRegister(instruction); - uint32_t custom_data = - linker::Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(data_reg); - Label* bne_label = NewBakerReadBarrierPatch(custom_data); - - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(IP, 12); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(IP); - __ LoadFromOffset(kLoadWord, kBakerCcEntrypointRegister, TR, entry_point_offset); - __ AddConstant(data_reg, obj, data_offset); - - Label return_address; - __ AdrCode(LR, &return_address); - __ CmpConstant(kBakerCcEntrypointRegister, 0); - EmitPlaceholderBne(this, bne_label); - ScopedForce32Bit maybe_force_32bit(down_cast<Thumb2Assembler*>(GetAssembler())); - int old_position = GetAssembler()->GetBuffer()->GetPosition(); - __ ldr(ref_reg, Address(data_reg, index_reg, LSL, scale_factor)); - DCHECK(!needs_null_check); // The thunk cannot handle the null check. - GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); - __ Bind(&return_address); - DCHECK_EQ(old_position - GetAssembler()->GetBuffer()->GetPosition(), - BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); - return; - } - - // /* HeapReference<Object> */ ref = - // *(obj + data_offset + index * sizeof(HeapReference<Object>)) - GenerateReferenceLoadWithBakerReadBarrier( - instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); -} - -void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - Location temp, - bool needs_null_check) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to mark the reference. - // Then, in the slow path, check the gray bit in the lock word of - // the reference's holder (`obj`) to decide whether to mark `ref` or - // not. - // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() - // // Slow path. - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // } - // } else { - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // } - - Register temp_reg = temp.AsRegister<Register>(); - - // Slow path marking the object `ref` when the GC is marking. The - // entrypoint will already be loaded in `temp2`. - Location temp2 = Location::RegisterLocation(LR); - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM( - instruction, - ref, - obj, - offset, - index, - scale_factor, - needs_null_check, - temp_reg, - /* entrypoint */ temp2); - AddSlowPath(slow_path); - - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ LoadFromOffset(kLoadWord, temp2.AsRegister<Register>(), TR, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(temp2.AsRegister<Register>(), slow_path->GetEntryLabel()); - // Fast path: the GC is not marking: just load the reference. - GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); - __ Bind(slow_path->GetExitLabel()); -} - -void CodeGeneratorARM::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - Location field_offset, - Location temp, - bool needs_null_check, - Register temp2) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to update the reference - // field within `obj`. Then, in the slow path, check the gray bit - // in the lock word of the reference's holder (`obj`) to decide - // whether to mark `ref` and update the field or not. - // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp3` the read barrier mark entry point - // corresponding to register `ref`. If `temp3` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() - // // Slow path. - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // old_ref = ref; - // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // compareAndSwapObject(obj, field_offset, old_ref, ref); - // } - // } - - Register temp_reg = temp.AsRegister<Register>(); - - // Slow path updating the object reference at address `obj + - // field_offset` when the GC is marking. The entrypoint will already - // be loaded in `temp3`. - Location temp3 = Location::RegisterLocation(LR); - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM( - instruction, - ref, - obj, - /* offset */ 0u, - /* index */ field_offset, - /* scale_factor */ ScaleFactor::TIMES_1, - needs_null_check, - temp_reg, - temp2, - /* entrypoint */ temp3); - AddSlowPath(slow_path); - - // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel()); - // Fast path: the GC is not marking: nothing to do (the field is - // up-to-date, and we don't need to load the reference). - __ Bind(slow_path->GetExitLabel()); -} - -void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check) { - Register ref_reg = ref.AsRegister<Register>(); - - if (index.IsValid()) { - // Load types involving an "index": ArrayGet, - // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject - // intrinsics. - // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor)) - if (index.IsConstant()) { - size_t computed_offset = - (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; - __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); - } else { - // Handle the special case of the - // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject - // intrinsics, which use a register pair as index ("long - // offset"), of which only the low part contains data. - Register index_reg = index.IsRegisterPair() - ? index.AsRegisterPairLow<Register>() - : index.AsRegister<Register>(); - __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); - __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); - } - } else { - // /* HeapReference<mirror::Object> */ ref = *(obj + offset) - __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); - } - - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - - // Object* ref = ref_addr->AsMirrorPtr() - __ MaybeUnpoisonHeapReference(ref_reg); -} - -void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { - DCHECK(kEmitCompilerReadBarrier); - - // Insert a slow path based read barrier *after* the reference load. - // - // If heap poisoning is enabled, the unpoisoning of the loaded - // reference will be carried out by the runtime within the slow - // path. - // - // Note that `ref` currently does not get unpoisoned (when heap - // poisoning is enabled), which is alright as the `ref` argument is - // not used by the artReadBarrierSlow entry point. - // - // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) - ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); - AddSlowPath(slow_path); - - __ b(slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); -} - -void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { - if (kEmitCompilerReadBarrier) { - // Baker's read barriers shall be handled by the fast path - // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). - DCHECK(!kUseBakerReadBarrier); - // If heap poisoning is enabled, unpoisoning will be taken care of - // by the runtime within the slow path. - GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); - } else if (kPoisonHeapReferences) { - __ UnpoisonHeapReference(out.AsRegister<Register>()); - } -} - -void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, - Location out, - Location root) { - DCHECK(kEmitCompilerReadBarrier); - - // Insert a slow path based read barrier *after* the GC root load. - // - // Note that GC roots are not affected by heap poisoning, so we do - // not need to do anything special for this here. - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); - AddSlowPath(slow_path); - - __ b(slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); -} - -HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( - const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, - HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { - return desired_dispatch_info; -} - -Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, - Register temp) { - DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); - Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - if (!invoke->GetLocations()->Intrinsified()) { - return location.AsRegister<Register>(); - } - // For intrinsics we allow any location, so it may be on the stack. - if (!location.IsRegister()) { - __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); - return temp; - } - // For register locations, check if the register was saved. If so, get it from the stack. - // Note: There is a chance that the register was saved but not overwritten, so we could - // save one load. However, since this is just an intrinsic slow path we prefer this - // simple and more robust approach rather that trying to determine if that's the case. - SlowPathCode* slow_path = GetCurrentSlowPath(); - DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. - if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { - int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); - __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); - return temp; - } - return location.AsRegister<Register>(); -} - -void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { - Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. - switch (invoke->GetMethodLoadKind()) { - case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { - uint32_t offset = - GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); - // temp = thread->string_init_entrypoint - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); - break; - } - case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: - callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - break; - case HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative: { - DCHECK(GetCompilerOptions().IsBootImage()); - Register temp_reg = temp.AsRegister<Register>(); - PcRelativePatchInfo* labels = NewPcRelativeMethodPatch(invoke->GetTargetMethod()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp_reg, temp_reg, ShifterOperand(PC)); - break; - } - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: - __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); - break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { - HArmDexCacheArraysBase* base = - invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); - Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, - temp.AsRegister<Register>()); - int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); - break; - } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register method_reg; - Register reg = temp.AsRegister<Register>(); - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ LoadFromOffset(kLoadWord, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); - break; - } - } - - switch (invoke->GetCodePtrLocation()) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: - __ bl(GetFrameEntryLabel()); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: - // LR = callee_method->entry_point_from_quick_compiled_code_ - __ LoadFromOffset( - kLoadWord, LR, callee_method.AsRegister<Register>(), - ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); - // LR() - __ blx(LR); - break; - } - - DCHECK(!IsLeafMethod()); -} - -void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { - Register temp = temp_location.AsRegister<Register>(); - uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( - invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); - - // Use the calling convention instead of the location of the receiver, as - // intrinsics may have put the receiver in a different register. In the intrinsics - // slow path, the arguments have been moved to the right place, so here we are - // guaranteed that the receiver is the first register of the calling convention. - InvokeDexCallingConvention calling_convention; - Register receiver = calling_convention.GetRegisterAt(0); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - // /* HeapReference<Class> */ temp = receiver->klass_ - __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); - MaybeRecordImplicitNullCheck(invoke); - // Instead of simply (possibly) unpoisoning `temp` here, we should - // emit a read barrier for the previous class reference load. - // However this is not required in practice, as this is an - // intermediate/temporary reference and because the current - // concurrent copying collector keeps the from-space memory - // intact/accessible until the end of the marking phase (the - // concurrent copying collector may not in the future). - __ MaybeUnpoisonHeapReference(temp); - // temp = temp->GetMethodAt(method_offset); - uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArmPointerSize).Int32Value(); - __ LoadFromOffset(kLoadWord, temp, temp, method_offset); - // LR = temp->GetEntryPoint(); - __ LoadFromOffset(kLoadWord, LR, temp, entry_point); - // LR(); - __ blx(LR); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeMethodPatch( - MethodReference target_method) { - return NewPcRelativePatch(*target_method.dex_file, - target_method.dex_method_index, - &pc_relative_method_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewTypeBssEntryPatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( - const DexFile& dex_file, dex::StringIndex string_index) { - return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( - const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { - patches->emplace_back(dex_file, offset_or_index); - return &patches->back(); -} - -Label* CodeGeneratorARM::NewBakerReadBarrierPatch(uint32_t custom_data) { - baker_read_barrier_patches_.emplace_back(custom_data); - return &baker_read_barrier_patches_.back().label; -} - -Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { - return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); -} - -Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, - dex::StringIndex string_index, - Handle<mirror::String> handle) { - jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), - reinterpret_cast64<uint64_t>(handle.GetReference())); - return jit_string_patches_.GetOrCreate( - StringReference(&dex_file, string_index), - [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); -} - -Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, - dex::TypeIndex type_index, - Handle<mirror::Class> handle) { - jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), - reinterpret_cast64<uint64_t>(handle.GetReference())); - return jit_class_patches_.GetOrCreate( - TypeReference(&dex_file, type_index), - [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); -} - -template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> -inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( - const ArenaDeque<PcRelativePatchInfo>& infos, - ArenaVector<LinkerPatch>* linker_patches) { - for (const PcRelativePatchInfo& info : infos) { - const DexFile& dex_file = info.target_dex_file; - size_t offset_or_index = info.offset_or_index; - DCHECK(info.add_pc_label.IsBound()); - uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); - // Add MOVW patch. - DCHECK(info.movw_label.IsBound()); - uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); - linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); - // Add MOVT patch. - DCHECK(info.movt_label.IsBound()); - uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); - linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); - } -} - -void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { - DCHECK(linker_patches->empty()); - size_t size = - /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * pc_relative_method_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + - baker_read_barrier_patches_.size(); - linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); - if (GetCompilerOptions().IsBootImage()) { - EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, - linker_patches); - EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, - linker_patches); - EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, - linker_patches); - } else { - DCHECK(pc_relative_method_patches_.empty()); - DCHECK(pc_relative_type_patches_.empty()); - EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, - linker_patches); - } - EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, - linker_patches); - for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { - linker_patches->push_back(LinkerPatch::BakerReadBarrierBranchPatch(info.label.Position(), - info.custom_data)); - } - DCHECK_EQ(size, linker_patches->size()); -} - -Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { - return map->GetOrCreate( - value, - [this, value]() { return __ NewLiteral<uint32_t>(value); }); -} - -void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); - locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, - Location::RequiresRegister()); - locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); - locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { - LocationSummary* locations = instr->GetLocations(); - Register res = locations->Out().AsRegister<Register>(); - Register accumulator = - locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); - Register mul_left = - locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); - Register mul_right = - locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); - - if (instr->GetOpKind() == HInstruction::kAdd) { - __ mla(res, mul_left, mul_right, accumulator); - } else { - __ mls(res, mul_left, mul_right, accumulator); - } -} - -void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, this should be removed during prepare for register allocator. - LOG(FATAL) << "Unreachable"; -} - -void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, this should be removed during prepare for register allocator. - LOG(FATAL) << "Unreachable"; -} - -// Simple implementation of packed switch - generate cascaded compare/jumps. -void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && - codegen_->GetAssembler()->IsThumb()) { - locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. - if (switch_instr->GetStartValue() != 0) { - locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. - } - } -} - -void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { - int32_t lower_bound = switch_instr->GetStartValue(); - uint32_t num_entries = switch_instr->GetNumEntries(); - LocationSummary* locations = switch_instr->GetLocations(); - Register value_reg = locations->InAt(0).AsRegister<Register>(); - HBasicBlock* default_block = switch_instr->GetDefaultBlock(); - - if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { - // Create a series of compare/jumps. - Register temp_reg = IP; - // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store - // the immediate, because IP is used as the destination register. For the other - // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, - // and they can be encoded in the instruction without making use of IP register. - __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); - - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - // Jump to successors[0] if value == lower_bound. - __ b(codegen_->GetLabelOf(successors[0]), EQ); - int32_t last_index = 0; - for (; num_entries - last_index > 2; last_index += 2) { - __ AddConstantSetFlags(temp_reg, temp_reg, -2); - // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. - __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); - // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. - __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); - } - if (num_entries - last_index == 2) { - // The last missing case_value. - __ CmpConstant(temp_reg, 1); - __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); - } - - // And the default for any other value. - if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { - __ b(codegen_->GetLabelOf(default_block)); - } - } else { - // Create a table lookup. - Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); - - // Materialize a pointer to the switch table - std::vector<Label*> labels(num_entries); - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - for (uint32_t i = 0; i < num_entries; i++) { - labels[i] = codegen_->GetLabelOf(successors[i]); - } - JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); - - // Remove the bias. - Register key_reg; - if (lower_bound != 0) { - key_reg = locations->GetTemp(1).AsRegister<Register>(); - __ AddConstant(key_reg, value_reg, -lower_bound); - } else { - key_reg = value_reg; - } - - // Check whether the value is in the table, jump to default block if not. - __ CmpConstant(key_reg, num_entries - 1); - __ b(codegen_->GetLabelOf(default_block), Condition::HI); - - // Load the displacement from the table. - __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); - - // Dispatch is a direct add to the PC (for Thumb2). - __ EmitJumpTableDispatch(table, temp_reg); - } -} - -void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(base_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(base_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(base_reg, base_reg, ShifterOperand(PC)); -} - -void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { - if (!trg.IsValid()) { - DCHECK_EQ(type, Primitive::kPrimVoid); - return; - } - - DCHECK_NE(type, Primitive::kPrimVoid); - - Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); - if (return_loc.Equals(trg)) { - return; - } - - // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged - // with the last branch. - if (type == Primitive::kPrimLong) { - HParallelMove parallel_move(GetGraph()->GetArena()); - parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); - parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); - GetMoveResolver()->EmitNativeCode(¶llel_move); - } else if (type == Primitive::kPrimDouble) { - HParallelMove parallel_move(GetGraph()->GetArena()); - parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); - parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); - GetMoveResolver()->EmitNativeCode(¶llel_move); - } else { - // Let the parallel move resolver take care of all of this. - HParallelMove parallel_move(GetGraph()->GetArena()); - parallel_move.AddMove(return_loc, trg, type, nullptr); - GetMoveResolver()->EmitNativeCode(¶llel_move); - } -} - -void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { - LocationSummary* locations = instruction->GetLocations(); - if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { - uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( - instruction->GetIndex(), kArmPointerSize).SizeValue(); - __ LoadFromOffset(kLoadWord, - locations->Out().AsRegister<Register>(), - locations->InAt(0).AsRegister<Register>(), - method_offset); - } else { - uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( - instruction->GetIndex(), kArmPointerSize)); - __ LoadFromOffset(kLoadWord, - locations->Out().AsRegister<Register>(), - locations->InAt(0).AsRegister<Register>(), - mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); - __ LoadFromOffset(kLoadWord, - locations->Out().AsRegister<Register>(), - locations->Out().AsRegister<Register>(), - method_offset); - } -} - -static void PatchJitRootUse(uint8_t* code, - const uint8_t* roots_data, - Literal* literal, - uint64_t index_in_table) { - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = literal->GetLabel()->Position(); - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - uint8_t* data = code + literal_offset; - reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); -} - -void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { - for (const auto& entry : jit_string_patches_) { - const StringReference& string_reference = entry.first; - Literal* table_entry_literal = entry.second; - const auto it = jit_string_roots_.find(string_reference); - DCHECK(it != jit_string_roots_.end()); - uint64_t index_in_table = it->second; - PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table); - } - for (const auto& entry : jit_class_patches_) { - const TypeReference& type_reference = entry.first; - Literal* table_entry_literal = entry.second; - const auto it = jit_class_roots_.find(type_reference); - DCHECK(it != jit_class_roots_.end()); - uint64_t index_in_table = it->second; - PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table); - } -} - -#undef __ -#undef QUICK_ENTRY_POINT - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h deleted file mode 100644 index 5f37d3bff1..0000000000 --- a/compiler/optimizing/code_generator_arm.h +++ /dev/null @@ -1,694 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_H_ -#define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_H_ - -#include "base/enums.h" -#include "code_generator.h" -#include "dex_file_types.h" -#include "driver/compiler_options.h" -#include "nodes.h" -#include "string_reference.h" -#include "parallel_move_resolver.h" -#include "type_reference.h" -#include "utils/arm/assembler_thumb2.h" - -namespace art { -namespace arm { - -class CodeGeneratorARM; - -// Use a local definition to prevent copying mistakes. -static constexpr size_t kArmWordSize = static_cast<size_t>(kArmPointerSize); -static constexpr size_t kArmBitsPerWord = kArmWordSize * kBitsPerByte; - -static constexpr Register kParameterCoreRegisters[] = { R1, R2, R3 }; -static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); -static constexpr SRegister kParameterFpuRegisters[] = - { S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, S12, S13, S14, S15 }; -static constexpr size_t kParameterFpuRegistersLength = arraysize(kParameterFpuRegisters); - -static constexpr Register kArtMethodRegister = R0; - -static constexpr Register kRuntimeParameterCoreRegisters[] = { R0, R1, R2, R3 }; -static constexpr size_t kRuntimeParameterCoreRegistersLength = - arraysize(kRuntimeParameterCoreRegisters); -static constexpr SRegister kRuntimeParameterFpuRegisters[] = { S0, S1, S2, S3 }; -static constexpr size_t kRuntimeParameterFpuRegistersLength = - arraysize(kRuntimeParameterFpuRegisters); - -class SlowPathCodeARM : public SlowPathCode { - public: - explicit SlowPathCodeARM(HInstruction* instruction) : SlowPathCode(instruction) {} - - void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) FINAL; - void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) FINAL; - - private: - DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM); -}; - - -class InvokeRuntimeCallingConvention : public CallingConvention<Register, SRegister> { - public: - InvokeRuntimeCallingConvention() - : CallingConvention(kRuntimeParameterCoreRegisters, - kRuntimeParameterCoreRegistersLength, - kRuntimeParameterFpuRegisters, - kRuntimeParameterFpuRegistersLength, - kArmPointerSize) {} - - private: - DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention); -}; - -constexpr DRegister FromLowSToD(SRegister reg) { - DCHECK_EQ(reg % 2, 0); - return static_cast<DRegister>(reg / 2); -} - - -class InvokeDexCallingConvention : public CallingConvention<Register, SRegister> { - public: - InvokeDexCallingConvention() - : CallingConvention(kParameterCoreRegisters, - kParameterCoreRegistersLength, - kParameterFpuRegisters, - kParameterFpuRegistersLength, - kArmPointerSize) {} - - private: - DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention); -}; - -class InvokeDexCallingConventionVisitorARM : public InvokeDexCallingConventionVisitor { - public: - InvokeDexCallingConventionVisitorARM() {} - virtual ~InvokeDexCallingConventionVisitorARM() {} - - Location GetNextLocation(Primitive::Type type) OVERRIDE; - Location GetReturnLocation(Primitive::Type type) const OVERRIDE; - Location GetMethodLocation() const OVERRIDE; - - private: - InvokeDexCallingConvention calling_convention; - uint32_t double_index_ = 0; - - DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM); -}; - -class FieldAccessCallingConventionARM : public FieldAccessCallingConvention { - public: - FieldAccessCallingConventionARM() {} - - Location GetObjectLocation() const OVERRIDE { - return Location::RegisterLocation(R1); - } - Location GetFieldIndexLocation() const OVERRIDE { - return Location::RegisterLocation(R0); - } - Location GetReturnLocation(Primitive::Type type) const OVERRIDE { - return Primitive::Is64BitType(type) - ? Location::RegisterPairLocation(R0, R1) - : Location::RegisterLocation(R0); - } - Location GetSetValueLocation(Primitive::Type type, bool is_instance) const OVERRIDE { - return Primitive::Is64BitType(type) - ? Location::RegisterPairLocation(R2, R3) - : (is_instance - ? Location::RegisterLocation(R2) - : Location::RegisterLocation(R1)); - } - Location GetFpuLocation(Primitive::Type type) const OVERRIDE { - return Primitive::Is64BitType(type) - ? Location::FpuRegisterPairLocation(S0, S1) - : Location::FpuRegisterLocation(S0); - } - - private: - DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM); -}; - -class ParallelMoveResolverARM : public ParallelMoveResolverWithSwap { - public: - ParallelMoveResolverARM(ArenaAllocator* allocator, CodeGeneratorARM* codegen) - : ParallelMoveResolverWithSwap(allocator), codegen_(codegen) {} - - void EmitMove(size_t index) OVERRIDE; - void EmitSwap(size_t index) OVERRIDE; - void SpillScratch(int reg) OVERRIDE; - void RestoreScratch(int reg) OVERRIDE; - - ArmAssembler* GetAssembler() const; - - private: - void Exchange(Register reg, int mem); - void Exchange(int mem1, int mem2); - - CodeGeneratorARM* const codegen_; - - DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM); -}; - -class LocationsBuilderARM : public HGraphVisitor { - public: - LocationsBuilderARM(HGraph* graph, CodeGeneratorARM* codegen) - : HGraphVisitor(graph), codegen_(codegen) {} - -#define DECLARE_VISIT_INSTRUCTION(name, super) \ - void Visit##name(H##name* instr) OVERRIDE; - - FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) - -#undef DECLARE_VISIT_INSTRUCTION - - void VisitInstruction(HInstruction* instruction) OVERRIDE { - LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() - << " (id " << instruction->GetId() << ")"; - } - - private: - void HandleInvoke(HInvoke* invoke); - void HandleBitwiseOperation(HBinaryOperation* operation, Opcode opcode); - void HandleCondition(HCondition* condition); - void HandleIntegerRotate(LocationSummary* locations); - void HandleLongRotate(LocationSummary* locations); - void HandleShift(HBinaryOperation* operation); - void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info); - void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); - - Location ArithmeticZeroOrFpuRegister(HInstruction* input); - Location ArmEncodableConstantOrRegister(HInstruction* constant, Opcode opcode); - bool CanEncodeConstantAsImmediate(HConstant* input_cst, Opcode opcode); - bool CanEncodeConstantAsImmediate(uint32_t value, Opcode opcode, SetCc set_cc = kCcDontCare); - - CodeGeneratorARM* const codegen_; - InvokeDexCallingConventionVisitorARM parameter_visitor_; - - DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM); -}; - -class InstructionCodeGeneratorARM : public InstructionCodeGenerator { - public: - InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen); - -#define DECLARE_VISIT_INSTRUCTION(name, super) \ - void Visit##name(H##name* instr) OVERRIDE; - - FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) - -#undef DECLARE_VISIT_INSTRUCTION - - void VisitInstruction(HInstruction* instruction) OVERRIDE { - LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() - << " (id " << instruction->GetId() << ")"; - } - - ArmAssembler* GetAssembler() const { return assembler_; } - - private: - // Generate code for the given suspend check. If not null, `successor` - // is the block to branch to if the suspend check is not needed, and after - // the suspend call. - void GenerateSuspendCheck(HSuspendCheck* check, HBasicBlock* successor); - void GenerateClassInitializationCheck(SlowPathCodeARM* slow_path, Register class_reg); - void GenerateAndConst(Register out, Register first, uint32_t value); - void GenerateOrrConst(Register out, Register first, uint32_t value); - void GenerateEorConst(Register out, Register first, uint32_t value); - void GenerateAddLongConst(Location out, Location first, uint64_t value); - void HandleBitwiseOperation(HBinaryOperation* operation); - void HandleCondition(HCondition* condition); - void HandleIntegerRotate(LocationSummary* locations); - void HandleLongRotate(HRor* ror); - void HandleShift(HBinaryOperation* operation); - - void GenerateWideAtomicStore(Register addr, uint32_t offset, - Register value_lo, Register value_hi, - Register temp1, Register temp2, - HInstruction* instruction); - void GenerateWideAtomicLoad(Register addr, uint32_t offset, - Register out_lo, Register out_hi); - - void HandleFieldSet(HInstruction* instruction, - const FieldInfo& field_info, - bool value_can_be_null); - void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); - - // Generate a heap reference load using one register `out`: - // - // out <- *(out + offset) - // - // while honoring heap poisoning and/or read barriers (if any). - // - // Location `maybe_temp` is used when generating a read barrier and - // shall be a register in that case; it may be an invalid location - // otherwise. - void GenerateReferenceLoadOneRegister(HInstruction* instruction, - Location out, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option); - // Generate a heap reference load using two different registers - // `out` and `obj`: - // - // out <- *(obj + offset) - // - // while honoring heap poisoning and/or read barriers (if any). - // - // Location `maybe_temp` is used when generating a Baker's (fast - // path) read barrier and shall be a register in that case; it may - // be an invalid location otherwise. - void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, - Location out, - Location obj, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option); - // Generate a GC root reference load: - // - // root <- *(obj + offset) - // - // while honoring read barriers based on read_barrier_option. - void GenerateGcRootFieldLoad(HInstruction* instruction, - Location root, - Register obj, - uint32_t offset, - ReadBarrierOption read_barrier_option); - void GenerateTestAndBranch(HInstruction* instruction, - size_t condition_input_index, - Label* true_target, - Label* false_target); - void GenerateCompareTestAndBranch(HCondition* condition, - Label* true_target, - Label* false_target); - void DivRemOneOrMinusOne(HBinaryOperation* instruction); - void DivRemByPowerOfTwo(HBinaryOperation* instruction); - void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction); - void GenerateDivRemConstantIntegral(HBinaryOperation* instruction); - void HandleGoto(HInstruction* got, HBasicBlock* successor); - - ArmAssembler* const assembler_; - CodeGeneratorARM* const codegen_; - - DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM); -}; - -class CodeGeneratorARM : public CodeGenerator { - public: - CodeGeneratorARM(HGraph* graph, - const ArmInstructionSetFeatures& isa_features, - const CompilerOptions& compiler_options, - OptimizingCompilerStats* stats = nullptr); - virtual ~CodeGeneratorARM() {} - - void GenerateFrameEntry() OVERRIDE; - void GenerateFrameExit() OVERRIDE; - void Bind(HBasicBlock* block) OVERRIDE; - void MoveConstant(Location destination, int32_t value) OVERRIDE; - void MoveLocation(Location dst, Location src, Primitive::Type dst_type) OVERRIDE; - void AddLocationAsTemp(Location location, LocationSummary* locations) OVERRIDE; - - size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - - size_t GetWordSize() const OVERRIDE { - return kArmWordSize; - } - - size_t GetFloatingPointSpillSlotSize() const OVERRIDE { - // Allocated in S registers, which are word sized. - return kArmWordSize; - } - - HGraphVisitor* GetLocationBuilder() OVERRIDE { - return &location_builder_; - } - - HGraphVisitor* GetInstructionVisitor() OVERRIDE { - return &instruction_visitor_; - } - - ArmAssembler* GetAssembler() OVERRIDE { - return &assembler_; - } - - const ArmAssembler& GetAssembler() const OVERRIDE { - return assembler_; - } - - uintptr_t GetAddressOf(HBasicBlock* block) OVERRIDE { - return GetLabelOf(block)->Position(); - } - - void SetupBlockedRegisters() const OVERRIDE; - - void DumpCoreRegister(std::ostream& stream, int reg) const OVERRIDE; - void DumpFloatingPointRegister(std::ostream& stream, int reg) const OVERRIDE; - - ParallelMoveResolverARM* GetMoveResolver() OVERRIDE { - return &move_resolver_; - } - - InstructionSet GetInstructionSet() const OVERRIDE { - return InstructionSet::kThumb2; - } - - // Helper method to move a 32bits value between two locations. - void Move32(Location destination, Location source); - // Helper method to move a 64bits value between two locations. - void Move64(Location destination, Location source); - - void LoadOrStoreToOffset(Primitive::Type type, - Location loc, - Register base, - int32_t offset, - bool is_load, - Condition cond = AL); - - void LoadFromShiftedRegOffset(Primitive::Type type, - Location out_loc, - Register base, - Register reg_offset, - Condition cond = AL); - void StoreToShiftedRegOffset(Primitive::Type type, - Location out_loc, - Register base, - Register reg_offset, - Condition cond = AL); - - // Generate code to invoke a runtime entry point. - void InvokeRuntime(QuickEntrypointEnum entrypoint, - HInstruction* instruction, - uint32_t dex_pc, - SlowPathCode* slow_path = nullptr) OVERRIDE; - - // Generate code to invoke a runtime entry point, but do not record - // PC-related information in a stack map. - void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, - HInstruction* instruction, - SlowPathCode* slow_path); - - void GenerateInvokeRuntime(int32_t entry_point_offset); - - // Emit a write barrier. - void MarkGCCard(Register temp, Register card, Register object, Register value, bool can_be_null); - - void GenerateMemoryBarrier(MemBarrierKind kind); - - Label* GetLabelOf(HBasicBlock* block) const { - return CommonGetLabelOf<Label>(block_labels_, block); - } - - Label* GetFinalLabel(HInstruction* instruction, Label* final_label); - - void Initialize() OVERRIDE { - block_labels_ = CommonInitializeLabels<Label>(); - } - - void Finalize(CodeAllocator* allocator) OVERRIDE; - - const ArmInstructionSetFeatures& GetInstructionSetFeatures() const { - return isa_features_; - } - - bool NeedsTwoRegisters(Primitive::Type type) const OVERRIDE { - return type == Primitive::kPrimDouble || type == Primitive::kPrimLong; - } - - void ComputeSpillMask() OVERRIDE; - - Label* GetFrameEntryLabel() { return &frame_entry_label_; } - - // Check if the desired_string_load_kind is supported. If it is, return it, - // otherwise return a fall-back kind that should be used instead. - HLoadString::LoadKind GetSupportedLoadStringKind( - HLoadString::LoadKind desired_string_load_kind) OVERRIDE; - - // Check if the desired_class_load_kind is supported. If it is, return it, - // otherwise return a fall-back kind that should be used instead. - HLoadClass::LoadKind GetSupportedLoadClassKind( - HLoadClass::LoadKind desired_class_load_kind) OVERRIDE; - - // Check if the desired_dispatch_info is supported. If it is, return it, - // otherwise return a fall-back info that should be used instead. - HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( - const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, - HInvokeStaticOrDirect* invoke) OVERRIDE; - - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; - - void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; - - // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays - // and boot image strings/types. The only difference is the interpretation of the - // offset_or_index. The PC-relative address is loaded with three instructions, - // MOVW+MOVT to load the offset to base_reg and then ADD base_reg, PC. The offset - // is calculated from the ADD's effective PC, i.e. PC+4 on Thumb2. Though we - // currently emit these 3 instructions together, instruction scheduling could - // split this sequence apart, so we keep separate labels for each of them. - struct PcRelativePatchInfo { - PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) - : target_dex_file(dex_file), offset_or_index(off_or_idx) { } - PcRelativePatchInfo(PcRelativePatchInfo&& other) = default; - - const DexFile& target_dex_file; - // Either the dex cache array element offset or the string/type index. - uint32_t offset_or_index; - Label movw_label; - Label movt_label; - Label add_pc_label; - }; - - PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); - PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); - PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); - PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, - dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); - - // Add a new baker read barrier patch and return the label to be bound - // before the BNE instruction. - Label* NewBakerReadBarrierPatch(uint32_t custom_data); - - Literal* DeduplicateBootImageAddressLiteral(uint32_t address); - Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, - dex::StringIndex string_index, - Handle<mirror::String> handle); - Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, - dex::TypeIndex type_index, - Handle<mirror::Class> handle); - - void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; - - void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; - - // Maybe add the reserved entrypoint register as a temporary for field load. This temp - // is added only for AOT compilation if link-time generated thunks for fields are enabled. - void MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations); - - // Fast path implementation of ReadBarrier::Barrier for a heap - // reference field load when Baker's read barriers are used. - void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location temp, - bool needs_null_check); - // Fast path implementation of ReadBarrier::Barrier for a heap - // reference array load when Baker's read barriers are used. - void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t data_offset, - Location index, - Location temp, - bool needs_null_check); - // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier, - // GenerateArrayLoadWithBakerReadBarrier and some intrinsics. - // - // Load the object reference located at the address - // `obj + offset + (index << scale_factor)`, held by object `obj`, into - // `ref`, and mark it if needed. - void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - Location temp, - bool needs_null_check); - - // Generate code checking whether the the reference field at the - // address `obj + field_offset`, held by object `obj`, needs to be - // marked, and if so, marking it and updating the field within `obj` - // with the marked value. - // - // This routine is used for the implementation of the - // UnsafeCASObject intrinsic with Baker read barriers. - // - // This method has a structure similar to - // GenerateReferenceLoadWithBakerReadBarrier, but note that argument - // `ref` is only as a temporary here, and thus its value should not - // be used afterwards. - void UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - Location field_offset, - Location temp, - bool needs_null_check, - Register temp2); - - // Generate a heap reference load (with no read barrier). - void GenerateRawReferenceLoad(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check); - - // Generate a read barrier for a heap reference within `instruction` - // using a slow path. - // - // A read barrier for an object reference read from the heap is - // implemented as a call to the artReadBarrierSlow runtime entry - // point, which is passed the values in locations `ref`, `obj`, and - // `offset`: - // - // mirror::Object* artReadBarrierSlow(mirror::Object* ref, - // mirror::Object* obj, - // uint32_t offset); - // - // The `out` location contains the value returned by - // artReadBarrierSlow. - // - // When `index` is provided (i.e. for array accesses), the offset - // value passed to artReadBarrierSlow is adjusted to take `index` - // into account. - void GenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // If read barriers are enabled, generate a read barrier for a heap - // reference using a slow path. If heap poisoning is enabled, also - // unpoison the reference in `out`. - void MaybeGenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // Generate a read barrier for a GC root within `instruction` using - // a slow path. - // - // A read barrier for an object reference GC root is implemented as - // a call to the artReadBarrierForRootSlow runtime entry point, - // which is passed the value in location `root`: - // - // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root); - // - // The `out` location contains the value returned by - // artReadBarrierForRootSlow. - void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); - - void GenerateNop() OVERRIDE; - - void GenerateImplicitNullCheck(HNullCheck* instruction) OVERRIDE; - void GenerateExplicitNullCheck(HNullCheck* instruction) OVERRIDE; - - // `temp` is an extra temporary register that is used for some conditions; - // callers may not specify it, in which case the method will use a scratch - // register instead. - void GenerateConditionWithZero(IfCondition condition, - Register out, - Register in, - Register temp = kNoRegister); - - private: - Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp); - - using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; - using StringToLiteralMap = ArenaSafeMap<StringReference, - Literal*, - StringReferenceValueComparator>; - using TypeToLiteralMap = ArenaSafeMap<TypeReference, - Literal*, - TypeReferenceValueComparator>; - - struct BakerReadBarrierPatchInfo { - explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { } - - Label label; - uint32_t custom_data; - }; - - Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); - PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, - uint32_t offset_or_index, - ArenaDeque<PcRelativePatchInfo>* patches); - template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> - static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos, - ArenaVector<LinkerPatch>* linker_patches); - - // Labels for each block that will be compiled. - Label* block_labels_; // Indexed by block id. - Label frame_entry_label_; - LocationsBuilderARM location_builder_; - InstructionCodeGeneratorARM instruction_visitor_; - ParallelMoveResolverARM move_resolver_; - Thumb2Assembler assembler_; - const ArmInstructionSetFeatures& isa_features_; - - // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. - Uint32ToLiteralMap uint32_literals_; - // PC-relative patch info for each HArmDexCacheArraysBase. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. - ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; - // PC-relative type patch info for kBootImageLinkTimePcRelative. - ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; - // PC-relative type patch info for kBssEntry. - ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; - // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). - ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; - // Baker read barrier patch info. - ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_; - - // Patches for string literals in JIT compiled code. - StringToLiteralMap jit_string_patches_; - // Patches for class literals in JIT compiled code. - TypeToLiteralMap jit_class_patches_; - - DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM); -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_H_ diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index f2b312362f..7e5b1a0fd1 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -19,6 +19,8 @@ #include "arch/arm64/asm_support_arm64.h" #include "arch/arm64/instruction_set_features_arm64.h" #include "art_method.h" +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "code_generator_utils.h" #include "compiled_method.h" #include "entrypoints/quick/quick_entrypoints.h" @@ -78,7 +80,6 @@ using helpers::VIXLRegCodeFromART; using helpers::WRegisterFrom; using helpers::XRegisterFrom; -static constexpr int kCurrentMethodStackOffset = 0; // The compare/jump sequence will generate about (1.5 * num_entries + 3) instructions. While jump // table version generates 7 instructions and num_entries literals. Compare/jump sequence will // generates less code/data with a small num_entries. @@ -671,7 +672,9 @@ void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { // `ref`. // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 { protected: ReadBarrierMarkSlowPathBaseARM64(HInstruction* instruction, Location ref, Location entrypoint) @@ -715,7 +718,7 @@ class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 { } else { // Entrypoint is not already loaded, load from the thread. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg()); // This runtime call does not require a stack map. arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); } @@ -742,9 +745,10 @@ class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 { // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). // -// If `entrypoint` is a valid location it is assumed to already be -// holding the entrypoint. The case where the entrypoint is passed in -// is when the decision to mark is based on whether the GC is marking. +// Argument `entrypoint` must be a register location holding the read +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { public: ReadBarrierMarkSlowPathARM64(HInstruction* instruction, @@ -790,7 +794,9 @@ class ReadBarrierMarkSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { // reference (different from `ref`) in `obj.field`). // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { public: LoadReferenceWithBakerReadBarrierSlowPathARM64(HInstruction* instruction, @@ -802,7 +808,7 @@ class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlo bool needs_null_check, bool use_load_acquire, Register temp, - Location entrypoint) + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -946,20 +952,23 @@ class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlo // another object reference (different from `ref`) in `obj.field`). // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { public: - LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - size_t scale_factor, - bool needs_null_check, - bool use_load_acquire, - Register temp, - Location entrypoint) + LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64( + HInstruction* instruction, + Location ref, + Register obj, + uint32_t offset, + Location index, + size_t scale_factor, + bool needs_null_check, + bool use_load_acquire, + Register temp, + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -1449,8 +1458,8 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1654,7 +1663,7 @@ void CodeGeneratorARM64::SetupBlockedRegisters() const { // Blocked core registers: // lr : Runtime reserved. // tr : Runtime reserved. - // xSuspend : Runtime reserved. TODO: Unblock this when the runtime stops using it. + // mr : Runtime reserved. // ip1 : VIXL core temp. // ip0 : VIXL core temp. // @@ -4490,7 +4499,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStatic return desired_dispatch_info; } -void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorARM64::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { @@ -4519,46 +4529,33 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok // Load method address from literal pool. __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { // Add ADRP with its PC-relative DexCache access patch. - const DexFile& dex_file = invoke->GetDexFileForPcRelativeDexCache(); - uint32_t element_offset = invoke->GetDexCacheArrayOffset(); - vixl::aarch64::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); + MethodReference target_method(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()); + vixl::aarch64::Label* adrp_label = NewMethodBssEntryPatch(target_method); EmitAdrpPlaceholder(adrp_label, XRegisterFrom(temp)); // Add LDR with its PC-relative DexCache access patch. vixl::aarch64::Label* ldr_label = - NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); + NewMethodBssEntryPatch(target_method, adrp_label); EmitLdrOffsetPlaceholder(ldr_label, XRegisterFrom(temp), XRegisterFrom(temp)); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register reg = XRegisterFrom(temp); - Register method_reg; - if (current_method.IsRegister()) { - method_reg = XRegisterFrom(current_method); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset)); - } - - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ Ldr(reg.X(), - MemOperand(method_reg.X(), - ArtMethod::DexCacheResolvedMethodsOffset(kArm64PointerSize).Int32Value())); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache))); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } switch (invoke->GetCodePtrLocation()) { case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: - __ Bl(&frame_entry_label_); + { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. + ExactAssemblyScope eas(GetVIXLAssembler(), + kInstructionSize, + CodeBufferCheckScope::kExactSize); + __ bl(&frame_entry_label_); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + } break; case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // LR = callee_method->entry_point_from_quick_compiled_code_; @@ -4566,14 +4563,13 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok XRegisterFrom(callee_method), ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize).Int32Value())); { - // To ensure that the pc position is recorded immediately after the `blr` instruction - // BLR must be the last instruction emitted in this function. - // Recording the pc will occur right after returning from this function. + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize); // lr() __ blr(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } break; } @@ -4581,7 +4577,8 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok DCHECK(!IsLeafMethod()); } -void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) { +void CodeGeneratorARM64::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_in, SlowPathCode* slow_path) { // Use the calling convention instead of the location of the receiver, as // intrinsics may have put the receiver in a different register. In the intrinsics // slow path, the arguments have been moved to the right place, so here we are @@ -4615,12 +4612,11 @@ void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location te // lr = temp->GetEntryPoint(); __ Ldr(lr, MemOperand(temp, entry_point.SizeValue())); { - // To ensure that the pc position is recorded immediately after the `blr` instruction - // BLR should be the last instruction emitted in this function. - // Recording the pc will occur right after returning from this function. + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize); // lr(); __ blr(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } } @@ -4641,6 +4637,15 @@ vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeMethodPatch( &pc_relative_method_patches_); } +vixl::aarch64::Label* CodeGeneratorARM64::NewMethodBssEntryPatch( + MethodReference target_method, + vixl::aarch64::Label* adrp_label) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + adrp_label, + &method_bss_entry_patches_); +} + vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeTypePatch( const DexFile& dex_file, dex::TypeIndex type_index, @@ -4663,13 +4668,6 @@ vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeStringPatch( NewPcRelativePatch(dex_file, string_index.index_, adrp_label, &pc_relative_string_patches_); } -vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, - uint32_t element_offset, - vixl::aarch64::Label* adrp_label) { - return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_); -} - vixl::aarch64::Label* CodeGeneratorARM64::NewBakerReadBarrierPatch(uint32_t custom_data) { baker_read_barrier_patches_.emplace_back(custom_data); return &baker_read_barrier_patches_.back().label; @@ -4691,7 +4689,7 @@ vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativePatch( vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral( uint64_t address) { - return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address)); } vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLiteral( @@ -4754,19 +4752,13 @@ inline void CodeGeneratorARM64::EmitPcRelativeLinkerPatches( void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + pc_relative_method_patches_.size() + + method_bss_entry_patches_.size() + pc_relative_type_patches_.size() + type_bss_entry_patches_.size() + pc_relative_string_patches_.size() + baker_read_barrier_patches_.size(); linker_patches->reserve(size); - for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { - linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.GetLocation(), - &info.target_dex_file, - info.pc_insn_label->GetLocation(), - info.offset_or_index)); - } if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -4780,6 +4772,8 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patc EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { @@ -4789,9 +4783,8 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patc DCHECK_EQ(size, linker_patches->size()); } -vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value, - Uint32ToLiteralMap* map) { - return map->GetOrCreate( +vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value) { + return uint32_literals_.GetOrCreate( value, [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); }); } @@ -4817,7 +4810,6 @@ void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDir LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -4830,7 +4822,6 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( @@ -5938,20 +5929,17 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( // Baker's read barrier are used. if (kBakerReadBarrierLinkTimeThunksEnableForGcRoots && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded GC root or not. Instead, we - // load into `temp` (actually IP1) the read barrier mark introspection - // entrypoint. If `temp` is null, it means that `GetIsGcMarking()` is - // false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // // We use link-time generated thunks for the slow path. That thunk // checks the reference and jumps to the entrypoint if needed. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &return_address; // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { - // goto gc_root_thunk<root_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto gc_root_thunk<root_reg>(lr) // } // return_address: @@ -5963,11 +5951,6 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( linker::Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg.GetCode()); vixl::aarch64::Label* cbnz_label = codegen_->NewBakerReadBarrierPatch(custom_data); - // ip1 = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(ip1, MemOperand(tr, entry_point_offset)); EmissionCheckScope guard(GetVIXLAssembler(), 3 * vixl::aarch64::kInstructionSize); vixl::aarch64::Label return_address; __ adr(lr, &return_address); @@ -5978,36 +5961,26 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( "GC root LDR must be 2 instruction (8B) before the return address label."); __ ldr(root_reg, MemOperand(obj.X(), offset)); __ Bind(cbnz_label); - __ cbnz(ip1, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. __ Bind(&return_address); } else { - // Note that we do not actually check the value of - // `GetIsGcMarking()` to decide whether to mark the loaded GC - // root or not. Instead, we load into `temp` the read barrier - // mark entry point corresponding to register `root`. If `temp` - // is null, it means that `GetIsGcMarking()` is false, and vice - // versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. - // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // root = entrypoint(root); // root = ReadBarrier::Mark(root); // Entry point call. // } - // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. - Register temp = lr; - SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64( - instruction, root, /* entrypoint */ LocationFrom(temp)); + // Slow path marking the GC root `root`. The entrypoint will + // be loaded by the slow path code. + SlowPathCodeARM64* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root); codegen_->AddSlowPath(slow_path); - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ Ldr(temp, MemOperand(tr, entry_point_offset)); - // /* GcRoot<mirror::Object> */ root = *(obj + offset) if (fixup_label == nullptr) { __ Ldr(root_reg, MemOperand(obj, offset)); @@ -6022,9 +5995,7 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( "art::mirror::CompressedReference<mirror::Object> and int32_t " "have different sizes."); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ Cbnz(temp, slow_path->GetEntryLabel()); + __ Cbnz(mr, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } } else { @@ -6065,20 +6036,19 @@ void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* ins if (kBakerReadBarrierLinkTimeThunksEnableForFields && !use_load_acquire && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually IP1) the read barrier mark introspection - // entrypoint. If `temp` is null, it means that `GetIsGcMarking()` is - // false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto field_thunk<holder_reg, base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -6104,17 +6074,12 @@ void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* ins obj.GetCode()); vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); - // ip1 = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(ip1, MemOperand(tr, entry_point_offset)); EmissionCheckScope guard(GetVIXLAssembler(), (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); vixl::aarch64::Label return_address; __ adr(lr, &return_address); __ Bind(cbnz_label); - __ cbnz(ip1, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), "Field LDR must be 1 instruction (4B) before the return address label; " " 2 instructions (8B) for heap poisoning."); @@ -6160,20 +6125,19 @@ void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* ins if (kBakerReadBarrierLinkTimeThunksEnableForArrays && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually IP1) the read barrier mark introspection - // entrypoint. If `temp` is null, it means that `GetIsGcMarking()` is - // false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto array_thunk<base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -6193,18 +6157,13 @@ void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* ins linker::Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(temp.GetCode()); vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); - // ip1 = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(ip1, MemOperand(tr, entry_point_offset)); __ Add(temp.X(), obj.X(), Operand(data_offset)); EmissionCheckScope guard(GetVIXLAssembler(), (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); vixl::aarch64::Label return_address; __ adr(lr, &return_address); __ Bind(cbnz_label); - __ cbnz(ip1, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. static_assert(BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), "Array LDR must be 1 instruction (4B) before the return address label; " " 2 instructions (8B) for heap poisoning."); @@ -6248,35 +6207,28 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* // `instruction->IsArrayGet()` => `!use_load_acquire`. DCHECK(!instruction->IsArrayGet() || !use_load_acquire); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to mark the reference. - // Then, in the slow path, check the gray bit in the lock word of - // the reference's holder (`obj`) to decide whether to mark `ref` or - // not. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<mirror::Object> ref = *src; // Original reference load. // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // } // } else { // HeapReference<mirror::Object> ref = *src; // Original reference load. // } // Slow path marking the object `ref` when the GC is marking. The - // entrypoint will already be loaded in `temp2`. - Register temp2 = lr; - Location temp2_loc = LocationFrom(temp2); + // entrypoint will be loaded by the slow path code. SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM64( instruction, @@ -6287,19 +6239,10 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* scale_factor, needs_null_check, use_load_acquire, - temp, - /* entrypoint */ temp2_loc); + temp); AddSlowPath(slow_path); - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ Ldr(temp2, MemOperand(tr, entry_point_offset)); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ Cbnz(temp2, slow_path->GetEntryLabel()); + __ Cbnz(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: just load the reference. GenerateRawReferenceLoad( instruction, ref, obj, offset, index, scale_factor, needs_null_check, use_load_acquire); @@ -6320,19 +6263,14 @@ void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* // `instruction->IsArrayGet()` => `!use_load_acquire`. DCHECK(!instruction->IsArrayGet() || !use_load_acquire); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to update the reference - // field within `obj`. Then, in the slow path, check the gray bit - // in the lock word of the reference's holder (`obj`) to decide - // whether to mark `ref` and update the field or not. - // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to update the reference field within `obj`. Then, in the + // slow path, check the gray bit in the lock word of the reference's + // holder (`obj`) to decide whether to mark `ref` and update the + // field or not. // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering @@ -6340,15 +6278,14 @@ void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // old_ref = ref; - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // compareAndSwapObject(obj, field_offset, old_ref, ref); // } // } // Slow path updating the object reference at address `obj + field_offset` - // when the GC is marking. The entrypoint will already be loaded in `temp2`. - Register temp2 = lr; - Location temp2_loc = LocationFrom(temp2); + // when the GC is marking. The entrypoint will be loaded by the slow path code. SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64( instruction, @@ -6359,19 +6296,10 @@ void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* /* scale_factor */ 0u /* "times 1" */, needs_null_check, use_load_acquire, - temp, - /* entrypoint */ temp2_loc); + temp); AddSlowPath(slow_path); - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ Ldr(temp2, MemOperand(tr, entry_point_offset)); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ Cbnz(temp2, slow_path->GetEntryLabel()); + __ Cbnz(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: nothing to do (the field is // up-to-date, and we don't need to load the reference). __ Bind(slow_path->GetExitLabel()); diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 747fc9f0b1..584eead81b 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -70,21 +70,32 @@ static const vixl::aarch64::FPRegister kParameterFPRegisters[] = { }; static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters); -// Thread Register +// Thread Register. const vixl::aarch64::Register tr = vixl::aarch64::x19; +// Marking Register. +const vixl::aarch64::Register mr = vixl::aarch64::x20; // Method register on invoke. static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0; const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0, vixl::aarch64::ip1); const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31); -const vixl::aarch64::CPURegList runtime_reserved_core_registers(tr, vixl::aarch64::lr); - -// Callee-saved registers AAPCS64 (without x19 - Thread Register) -const vixl::aarch64::CPURegList callee_saved_core_registers(vixl::aarch64::CPURegister::kRegister, - vixl::aarch64::kXRegSize, - vixl::aarch64::x20.GetCode(), - vixl::aarch64::x30.GetCode()); +const vixl::aarch64::CPURegList runtime_reserved_core_registers = + vixl::aarch64::CPURegList( + tr, + // Reserve X20 as Marking Register when emitting Baker read barriers. + ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) ? mr : vixl::aarch64::NoCPUReg), + vixl::aarch64::lr); + +// Callee-save registers AAPCS64, without x19 (Thread Register) (nor +// x20 (Marking Register) when emitting Baker read barriers). +const vixl::aarch64::CPURegList callee_saved_core_registers( + vixl::aarch64::CPURegister::kRegister, + vixl::aarch64::kXRegSize, + ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) + ? vixl::aarch64::x21.GetCode() + : vixl::aarch64::x20.GetCode()), + vixl::aarch64::x30.GetCode()); const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kFPRegister, vixl::aarch64::kDRegSize, vixl::aarch64::d8.GetCode(), @@ -540,8 +551,10 @@ class CodeGeneratorARM64 : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED, Primitive::Type type ATTRIBUTE_UNUSED) OVERRIDE { @@ -555,6 +568,13 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Label* NewPcRelativeMethodPatch(MethodReference target_method, vixl::aarch64::Label* adrp_label = nullptr); + // Add a new .bss entry method patch for an instruction and return + // the label to be bound before the instruction. The instruction will be + // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` + // pointing to the associated ADRP patch label). + vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method, + vixl::aarch64::Label* adrp_label = nullptr); + // Add a new PC-relative type patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing @@ -579,15 +599,6 @@ class CodeGeneratorARM64 : public CodeGenerator { dex::StringIndex string_index, vixl::aarch64::Label* adrp_label = nullptr); - // Add a new PC-relative dex cache array patch for an instruction and return - // the label to be bound before the instruction. The instruction will be - // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` - // pointing to the associated ADRP patch label). - vixl::aarch64::Label* NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, - uint32_t element_offset, - vixl::aarch64::Label* adrp_label = nullptr); - // Add a new baker read barrier patch and return the label to be bound // before the CBNZ instruction. vixl::aarch64::Label* NewBakerReadBarrierPatch(uint32_t custom_data); @@ -739,8 +750,7 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Literal<uint32_t>*, TypeReferenceValueComparator>; - vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, - Uint32ToLiteralMap* map); + vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value); vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value); // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays @@ -791,10 +801,10 @@ class CodeGeneratorARM64 : public CodeGenerator { Uint32ToLiteralMap uint32_literals_; // Deduplication map for 64-bit literals, used for non-patchable method address or method code. Uint64ToLiteralMap uint64_literals_; - // PC-relative DexCache access info. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 93cbc3b17c..b9d4700511 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -19,6 +19,8 @@ #include "arch/arm/asm_support_arm.h" #include "arch/arm/instruction_set_features_arm.h" #include "art_method.h" +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "code_generator_utils.h" #include "common_arm.h" #include "compiled_method.h" @@ -76,7 +78,6 @@ static bool ExpectedPairLayout(Location location) { // Use a local definition to prevent copying mistakes. static constexpr size_t kArmWordSize = static_cast<size_t>(kArmPointerSize); static constexpr size_t kArmBitsPerWord = kArmWordSize * kBitsPerByte; -static constexpr int kCurrentMethodStackOffset = 0; static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; // Reference load (except object array loads) is using LDR Rt, [Rn, #offset] which can handle @@ -739,7 +740,9 @@ class ArraySetSlowPathARMVIXL : public SlowPathCodeARMVIXL { // `ref`. // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL { protected: ReadBarrierMarkSlowPathBaseARMVIXL(HInstruction* instruction, Location ref, Location entrypoint) @@ -785,7 +788,7 @@ class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL { } else { // Entrypoint is not already loaded, load from the thread. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode()); + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode()); // This runtime call does not require a stack map. arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); } @@ -812,9 +815,10 @@ class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL { // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). // -// If `entrypoint` is a valid location it is assumed to already be -// holding the entrypoint. The case where the entrypoint is passed in -// is when the decision to mark is based on whether the GC is marking. +// Argument `entrypoint` must be a register location holding the read +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { public: ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction, @@ -860,7 +864,9 @@ class ReadBarrierMarkSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL // reference (different from `ref`) in `obj.field`). // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { public: LoadReferenceWithBakerReadBarrierSlowPathARMVIXL(HInstruction* instruction, @@ -871,7 +877,7 @@ class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkS ScaleFactor scale_factor, bool needs_null_check, vixl32::Register temp, - Location entrypoint) + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -1005,22 +1011,24 @@ class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkS // hold the same to-space reference (unless another thread installed // another object reference (different from `ref`) in `obj.field`). // -// // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { public: - LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction, - Location ref, - vixl32::Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check, - vixl32::Register temp1, - vixl32::Register temp2, - Location entrypoint) + LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL( + HInstruction* instruction, + Location ref, + vixl32::Register obj, + uint32_t offset, + Location index, + ScaleFactor scale_factor, + bool needs_null_check, + vixl32::Register temp1, + vixl32::Register temp2, + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -1287,8 +1295,8 @@ class ReadBarrierForHeapReferenceSlowPathARMVIXL : public SlowPathCodeARMVIXL { DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg.GetCode())); if (codegen->IsCoreCalleeSaveRegister(index_reg.GetCode())) { // We are about to change the value of `index_reg` (see the - // calls to art::arm::Thumb2Assembler::Lsl and - // art::arm::Thumb2Assembler::AddConstant below), but it has + // calls to art::arm::ArmVIXLMacroAssembler::Lsl and + // art::arm::ArmVIXLMacroAssembler::Add below), but it has // not been saved by the previous call to // art::SlowPathCode::SaveLiveRegisters, as it is a // callee-save register -- @@ -2309,7 +2317,8 @@ static void GenerateConditionLong(HCondition* cond, CodeGeneratorARMVIXL* codege } } -static void GenerateConditionIntegralOrNonPrimitive(HCondition* cond, CodeGeneratorARMVIXL* codegen) { +static void GenerateConditionIntegralOrNonPrimitive(HCondition* cond, + CodeGeneratorARMVIXL* codegen) { const Primitive::Type type = cond->GetLeft()->GetType(); DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; @@ -2500,8 +2509,8 @@ CodeGeneratorARMVIXL::CodeGeneratorARMVIXL(HGraph* graph, isa_features_(isa_features), uint32_literals_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -2575,6 +2584,11 @@ void CodeGeneratorARMVIXL::SetupBlockedRegisters() const { blocked_core_registers_[LR] = true; blocked_core_registers_[PC] = true; + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Reserve marking register. + blocked_core_registers_[MR] = true; + } + // Reserve thread register. blocked_core_registers_[TR] = true; @@ -2658,14 +2672,6 @@ void CodeGeneratorARMVIXL::GenerateFrameEntry() { GetAssembler()->cfi().RelOffsetForMany(DWARFReg(s0), 0, fpu_spill_mask_, kArmWordSize); } - if (GetGraph()->HasShouldDeoptimizeFlag()) { - UseScratchRegisterScope temps(GetVIXLAssembler()); - vixl32::Register temp = temps.Acquire(); - // Initialize should_deoptimize flag to 0. - __ Mov(temp, 0); - GetAssembler()->StoreToOffset(kStoreWord, temp, sp, -kShouldDeoptimizeFlagSize); - } - int adjust = GetFrameSize() - FrameEntrySpillSize(); __ Sub(sp, sp, adjust); GetAssembler()->cfi().AdjustCFAOffset(adjust); @@ -2676,6 +2682,14 @@ void CodeGeneratorARMVIXL::GenerateFrameEntry() { if (RequiresCurrentMethod()) { GetAssembler()->StoreToOffset(kStoreWord, kMethodRegister, sp, 0); } + + if (GetGraph()->HasShouldDeoptimizeFlag()) { + UseScratchRegisterScope temps(GetVIXLAssembler()); + vixl32::Register temp = temps.Acquire(); + // Initialize should_deoptimize flag to 0. + __ Mov(temp, 0); + GetAssembler()->StoreToOffset(kStoreWord, temp, sp, GetStackOffsetOfShouldDeoptimizeFlag()); + } } void CodeGeneratorARMVIXL::GenerateFrameExit() { @@ -3650,18 +3664,10 @@ void LocationsBuilderARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* i IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_); if (intrinsic.TryDispatch(invoke)) { - if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); - } return; } HandleInvoke(invoke); - - // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. - if (invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); - } } static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARMVIXL* codegen) { @@ -3685,7 +3691,6 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrD LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderARMVIXL::HandleInvoke(HInvoke* invoke) { @@ -3708,7 +3713,6 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) } codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); DCHECK(!codegen_->IsLeafMethod()); } @@ -8540,20 +8544,17 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( // Baker's read barrier are used. if (kBakerReadBarrierLinkTimeThunksEnableForGcRoots && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded GC root or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // // We use link-time generated thunks for the slow path. That thunk // checks the reference and jumps to the entrypoint if needed. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &return_address; // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { - // goto gc_root_thunk<root_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto gc_root_thunk<root_reg>(lr) // } // return_address: @@ -8564,18 +8565,10 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( root_reg.GetCode(), narrow); vixl32::Label* bne_label = codegen_->NewBakerReadBarrierPatch(custom_data); - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(kBakerCcEntrypointRegister, MemOperand(tr, entry_point_offset)); - - vixl::EmissionCheckScope guard(GetVIXLAssembler(), - 4 * vixl32::kMaxInstructionSizeInBytes); + vixl::EmissionCheckScope guard(GetVIXLAssembler(), 4 * vixl32::kMaxInstructionSizeInBytes); vixl32::Label return_address; EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(kBakerCcEntrypointRegister, Operand(0)); + __ cmp(mr, Operand(0)); // Currently the offset is always within range. If that changes, // we shall have to split the load the same way as for fields. DCHECK_LT(offset, kReferenceLoadMinFarOffset); @@ -8587,34 +8580,23 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( narrow ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_OFFSET); } else { - // Note that we do not actually check the value of - // `GetIsGcMarking()` to decide whether to mark the loaded GC - // root or not. Instead, we load into `temp` the read barrier - // mark entry point corresponding to register `root`. If `temp` - // is null, it means that `GetIsGcMarking()` is false, and vice - // versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. - // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // root = entrypoint(root); // root = ReadBarrier::Mark(root); // Entry point call. // } - // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. - Location temp = LocationFrom(lr); + // Slow path marking the GC root `root`. The entrypoint will + // be loaded by the slow path code. SlowPathCodeARMVIXL* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL( - instruction, root, /* entrypoint */ temp); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(instruction, root); codegen_->AddSlowPath(slow_path); - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, entry_point_offset); - // /* GcRoot<mirror::Object> */ root = *(obj + offset) GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset); static_assert( @@ -8625,9 +8607,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( "art::mirror::CompressedReference<mirror::Object> and int32_t " "have different sizes."); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(RegisterFrom(temp), slow_path->GetEntryLabel()); + __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } } else { @@ -8668,20 +8648,19 @@ void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* i if (kBakerReadBarrierLinkTimeThunksEnableForFields && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto field_thunk<holder_reg, base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -8710,19 +8689,12 @@ void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* i base.GetCode(), obj.GetCode(), narrow); vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(kBakerCcEntrypointRegister, MemOperand(tr, entry_point_offset)); - vixl::EmissionCheckScope guard( GetVIXLAssembler(), (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); vixl32::Label return_address; EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(kBakerCcEntrypointRegister, Operand(0)); + __ cmp(mr, Operand(0)); EmitPlaceholderBne(this, bne_label); ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); __ ldr(EncodingSize(narrow ? Narrow : Wide), ref_reg, MemOperand(base, offset)); @@ -8769,20 +8741,19 @@ void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* i if (kBakerReadBarrierLinkTimeThunksEnableForArrays && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto array_thunk<base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -8802,20 +8773,13 @@ void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* i linker::Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(data_reg.GetCode()); vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(kBakerCcEntrypointRegister, MemOperand(tr, entry_point_offset)); __ Add(data_reg, obj, Operand(data_offset)); - vixl::EmissionCheckScope guard( GetVIXLAssembler(), (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); vixl32::Label return_address; EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(kBakerCcEntrypointRegister, Operand(0)); + __ cmp(mr, Operand(0)); EmitPlaceholderBne(this, bne_label); ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); __ ldr(ref_reg, MemOperand(data_reg, index_reg, vixl32::LSL, scale_factor)); @@ -8847,26 +8811,21 @@ void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier(HInstructio DCHECK(kEmitCompilerReadBarrier); DCHECK(kUseBakerReadBarrier); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to mark the reference. - // Then, in the slow path, check the gray bit in the lock word of - // the reference's holder (`obj`) to decide whether to mark `ref` or - // not. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<mirror::Object> ref = *src; // Original reference load. // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // } // } else { // HeapReference<mirror::Object> ref = *src; // Original reference load. @@ -8875,30 +8834,13 @@ void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier(HInstructio vixl32::Register temp_reg = RegisterFrom(temp); // Slow path marking the object `ref` when the GC is marking. The - // entrypoint will already be loaded in `temp2`. - Location temp2 = LocationFrom(lr); + // entrypoint will be loaded by the slow path code. SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARMVIXL( - instruction, - ref, - obj, - offset, - index, - scale_factor, - needs_null_check, - temp_reg, - /* entrypoint */ temp2); + instruction, ref, obj, offset, index, scale_factor, needs_null_check, temp_reg); AddSlowPath(slow_path); - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp2), tr, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(RegisterFrom(temp2), slow_path->GetEntryLabel()); + __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: just load the reference. GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); __ Bind(slow_path->GetExitLabel()); @@ -8914,19 +8856,14 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction DCHECK(kEmitCompilerReadBarrier); DCHECK(kUseBakerReadBarrier); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to update the reference - // field within `obj`. Then, in the slow path, check the gray bit - // in the lock word of the reference's holder (`obj`) to decide - // whether to mark `ref` and update the field or not. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to update the reference field within `obj`. Then, in the + // slow path, check the gray bit in the lock word of the reference's + // holder (`obj`) to decide whether to mark `ref` and update the + // field or not. // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp3` the read barrier mark entry point - // corresponding to register `ref`. If `temp3` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering @@ -8934,7 +8871,8 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // old_ref = ref; - // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // compareAndSwapObject(obj, field_offset, old_ref, ref); // } // } @@ -8942,8 +8880,7 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction vixl32::Register temp_reg = RegisterFrom(temp); // Slow path updating the object reference at address `obj + field_offset` - // when the GC is marking. The entrypoint will already be loaded in `temp3`. - Location temp3 = LocationFrom(lr); + // when the GC is marking. The entrypoint will be loaded by the slow path code. SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL( instruction, @@ -8954,19 +8891,10 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction /* scale_factor */ ScaleFactor::TIMES_1, needs_null_check, temp_reg, - temp2, - /* entrypoint */ temp3); + temp2); AddSlowPath(slow_path); - // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp3), tr, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(RegisterFrom(temp3), slow_path->GetEntryLabel()); + __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: nothing to do (the field is // up-to-date, and we don't need to load the reference). __ Bind(slow_path->GetExitLabel()); @@ -9066,7 +8994,7 @@ void CodeGeneratorARMVIXL::MaybeGenerateReadBarrierSlow(HInstruction* instructio Location index) { if (kEmitCompilerReadBarrier) { // Baker's read barriers shall be handled by the fast path - // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). + // (CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier). DCHECK(!kUseBakerReadBarrier); // If heap poisoning is enabled, unpoisoning will be taken care of // by the runtime within the slow path. @@ -9127,7 +9055,7 @@ vixl32::Register CodeGeneratorARMVIXL::GetInvokeStaticOrDirectExtraParameter( } void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( - HInvokeStaticOrDirect* invoke, Location temp) { + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { @@ -9150,44 +9078,30 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ Mov(RegisterFrom(temp), Operand::From(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { - HArmDexCacheArraysBase* base = - invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); - vixl32::Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, RegisterFrom(temp)); - int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), base_reg, offset); + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { + PcRelativePatchInfo* labels = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); + vixl32::Register temp_reg = RegisterFrom(temp); + EmitMovwMovtPlaceholder(labels, temp_reg); + GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, temp_reg, /* offset*/ 0); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - vixl32::Register method_reg; - vixl32::Register reg = RegisterFrom(temp); - if (current_method.IsRegister()) { - method_reg = RegisterFrom(current_method); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - GetAssembler()->LoadFromOffset(kLoadWord, reg, sp, kCurrentMethodStackOffset); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - GetAssembler()->LoadFromOffset( - kLoadWord, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - GetAssembler()->LoadFromOffset( - kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } switch (invoke->GetCodePtrLocation()) { case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: - __ Bl(GetFrameEntryLabel()); + { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k32BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ bl(GetFrameEntryLabel()); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + } break; case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // LR = callee_method->entry_point_from_quick_compiled_code_ @@ -9197,12 +9111,14 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( RegisterFrom(callee_method), ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. ExactAssemblyScope aas(GetVIXLAssembler(), vixl32::k16BitT32InstructionSizeInBytes, CodeBufferCheckScope::kExactSize); // LR() __ blx(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } break; } @@ -9210,7 +9126,8 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( DCHECK(!IsLeafMethod()); } -void CodeGeneratorARMVIXL::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { +void CodeGeneratorARMVIXL::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { vixl32::Register temp = RegisterFrom(temp_location); uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); @@ -9246,15 +9163,16 @@ void CodeGeneratorARMVIXL::GenerateVirtualCall(HInvokeVirtual* invoke, Location GetAssembler()->LoadFromOffset(kLoadWord, temp, temp, method_offset); // LR = temp->GetEntryPoint(); GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, entry_point); - // LR(); - // This `blx` *must* be the *last* instruction generated by this stub, so that calls to - // `RecordPcInfo()` immediately following record the correct pc. Use a scope to help guarantee - // that. - // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - ExactAssemblyScope aas(GetVIXLAssembler(), - vixl32::k16BitT32InstructionSizeInBytes, - CodeBufferCheckScope::kExactSize); - __ blx(lr); + { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. + // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k16BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kExactSize); + // LR(); + __ blx(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + } } CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeMethodPatch( @@ -9264,6 +9182,13 @@ CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeMe &pc_relative_method_patches_); } +CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewMethodBssEntryPatch( + MethodReference target_method) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + &method_bss_entry_patches_); +} + CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeTypePatch( const DexFile& dex_file, dex::TypeIndex type_index) { return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); @@ -9279,11 +9204,6 @@ CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeSt return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); } -CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); -} - CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativePatch( const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { patches->emplace_back(dex_file, offset_or_index); @@ -9347,15 +9267,13 @@ inline void CodeGeneratorARMVIXL::EmitPcRelativeLinkerPatches( void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_method_patches_.size() + + /* MOVW+MOVT for each entry */ 2u * method_bss_entry_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + baker_read_barrier_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -9369,6 +9287,8 @@ void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pa EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { @@ -9518,17 +9438,6 @@ void InstructionCodeGeneratorARMVIXL::VisitPackedSwitch(HPackedSwitch* switch_in } } } -void LocationsBuilderARMVIXL::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARMVIXL::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - vixl32::Register base_reg = OutputRegister(base); - CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); - codegen_->EmitMovwMovtPlaceholder(labels, base_reg); -} // Copy the result of a call into the given target. void CodeGeneratorARMVIXL::MoveFromReturnRegister(Location trg, Primitive::Type type) { diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index f6e4de33a8..01cf287f29 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -35,13 +35,6 @@ #include "aarch32/macro-assembler-aarch32.h" #pragma GCC diagnostic pop -// Default to use the VIXL-based backend on ARM. -#ifdef ART_USE_OLD_ARM_BACKEND -static constexpr bool kArmUseVIXL32 = false; -#else -static constexpr bool kArmUseVIXL32 = true; -#endif - namespace art { namespace arm { @@ -80,12 +73,16 @@ static const vixl::aarch32::Register kMethodRegister = vixl::aarch32::r0; static const vixl::aarch32::Register kCoreAlwaysSpillRegister = vixl::aarch32::r5; -// Callee saves core registers r5, r6, r7, r8, r10, r11, and lr. +// Callee saves core registers r5, r6, r7, r8 (except when emitting Baker +// read barriers, where it is used as Marking Register), r10, r11, and lr. static const vixl::aarch32::RegisterList kCoreCalleeSaves = vixl::aarch32::RegisterList::Union( vixl::aarch32::RegisterList(vixl::aarch32::r5, vixl::aarch32::r6, - vixl::aarch32::r7, - vixl::aarch32::r8), + vixl::aarch32::r7), + // Do not consider r8 as a callee-save register with Baker read barriers. + ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) + ? vixl::aarch32::RegisterList() + : vixl::aarch32::RegisterList(vixl::aarch32::r8)), vixl::aarch32::RegisterList(vixl::aarch32::r10, vixl::aarch32::r11, vixl::aarch32::lr)); @@ -408,6 +405,17 @@ class InstructionCodeGeneratorARMVIXL : public InstructionCodeGenerator { void GenerateDivRemConstantIntegral(HBinaryOperation* instruction); void HandleGoto(HInstruction* got, HBasicBlock* successor); + vixl::aarch32::MemOperand VecAddress( + HVecMemoryOperation* instruction, + // This function may acquire a scratch register. + vixl::aarch32::UseScratchRegisterScope* temps_scope, + /*out*/ vixl32::Register* scratch); + vixl::aarch32::AlignedMemOperand VecAddressUnaligned( + HVecMemoryOperation* instruction, + // This function may acquire a scratch register. + vixl::aarch32::UseScratchRegisterScope* temps_scope, + /*out*/ vixl32::Register* scratch); + ArmVIXLAssembler* const assembler_; CodeGeneratorARMVIXL* const codegen_; @@ -538,8 +546,10 @@ class CodeGeneratorARMVIXL : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; @@ -564,12 +574,11 @@ class CodeGeneratorARMVIXL : public CodeGenerator { }; PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); + PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method); PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); // Add a new baker read barrier patch and return the label to be bound // before the BNE instruction. @@ -764,10 +773,10 @@ class CodeGeneratorARMVIXL : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // PC-relative patch info for each HArmDexCacheArraysBase. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 951d75a708..23d188d630 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -40,10 +40,6 @@ namespace mips { static constexpr int kCurrentMethodStackOffset = 0; static constexpr Register kMethodRegisterArgument = A0; -// We'll maximize the range of a single load instruction for dex cache array accesses -// by aligning offset -32768 with the offset of the first used element. -static constexpr uint32_t kDexCacheArrayLwOffset = 0x8000; - Location MipsReturnLocation(Primitive::Type return_type) { switch (return_type) { case Primitive::kPrimBoolean: @@ -212,8 +208,13 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { LoadClassSlowPathMIPS(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, - bool do_clinit) - : SlowPathCodeMIPS(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { + bool do_clinit, + const CodeGeneratorMIPS::PcRelativePatchInfo* bss_info_high = nullptr) + : SlowPathCodeMIPS(at), + cls_(cls), + dex_pc_(dex_pc), + do_clinit_(do_clinit), + bss_info_high_(bss_info_high) { DCHECK(at->IsLoadClass() || at->IsClinitCheck()); } @@ -221,8 +222,7 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { LocationSummary* locations = instruction_->GetLocations(); Location out = locations->Out(); CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); - const bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); - const bool r2_baker_or_no_read_barriers = !isR6 && (!kUseReadBarrier || kUseBakerReadBarrier); + const bool baker_or_no_read_barriers = (!kUseReadBarrier || kUseBakerReadBarrier); InvokeRuntimeCallingConvention calling_convention; DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); const bool is_load_class_bss_entry = @@ -232,7 +232,7 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. Register entry_address = kNoRegister; - if (is_load_class_bss_entry && r2_baker_or_no_read_barriers) { + if (is_load_class_bss_entry && baker_or_no_read_barriers) { Register temp = locations->GetTemp(0).AsRegister<Register>(); bool temp_is_a0 = (temp == calling_convention.GetRegisterAt(0)); // In the unlucky case that `temp` is A0, we preserve the address in `out` across the @@ -256,9 +256,18 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { } // For HLoadClass/kBssEntry, store the resolved class to the BSS entry. - if (is_load_class_bss_entry && r2_baker_or_no_read_barriers) { + if (is_load_class_bss_entry && baker_or_no_read_barriers) { // The class entry address was preserved in `entry_address` thanks to kSaveEverything. - __ StoreToOffset(kStoreWord, calling_convention.GetRegisterAt(0), entry_address, 0); + DCHECK(bss_info_high_); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + mips_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index, bss_info_high_); + bool reordering = __ SetReorder(false); + __ Bind(&info_low->label); + __ StoreToOffset(kStoreWord, + calling_convention.GetRegisterAt(0), + entry_address, + /* placeholder */ 0x5678); + __ SetReorder(reordering); } // Move the class to the desired location. @@ -272,14 +281,17 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { RestoreLiveRegisters(codegen, locations); // For HLoadClass/kBssEntry, store the resolved class to the BSS entry. - if (is_load_class_bss_entry && !r2_baker_or_no_read_barriers) { - // For non-Baker read barriers (or on R6), we need to re-calculate the address of + if (is_load_class_bss_entry && !baker_or_no_read_barriers) { + // For non-Baker read barriers we need to re-calculate the address of // the class entry. + const bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); Register base = isR6 ? ZERO : locations->InAt(0).AsRegister<Register>(); - CodeGeneratorMIPS::PcRelativePatchInfo* info = + CodeGeneratorMIPS::PcRelativePatchInfo* info_high = mips_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + mips_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index, info_high); bool reordering = __ SetReorder(false); - mips_codegen->EmitPcRelativeAddressPlaceholderHigh(info, TMP, base); + mips_codegen->EmitPcRelativeAddressPlaceholderHigh(info_high, TMP, base, info_low); __ StoreToOffset(kStoreWord, out.AsRegister<Register>(), TMP, /* placeholder */ 0x5678); __ SetReorder(reordering); } @@ -298,12 +310,17 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { // Whether to initialize the class. const bool do_clinit_; + // Pointer to the high half PC-relative patch info for HLoadClass/kBssEntry. + const CodeGeneratorMIPS::PcRelativePatchInfo* bss_info_high_; + DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathMIPS); }; class LoadStringSlowPathMIPS : public SlowPathCodeMIPS { public: - explicit LoadStringSlowPathMIPS(HLoadString* instruction) : SlowPathCodeMIPS(instruction) {} + explicit LoadStringSlowPathMIPS(HLoadString* instruction, + const CodeGeneratorMIPS::PcRelativePatchInfo* bss_info_high) + : SlowPathCodeMIPS(instruction), bss_info_high_(bss_info_high) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { DCHECK(instruction_->IsLoadString()); @@ -314,15 +331,14 @@ class LoadStringSlowPathMIPS : public SlowPathCodeMIPS { const dex::StringIndex string_index = load->GetStringIndex(); Register out = locations->Out().AsRegister<Register>(); CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); - const bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); - const bool r2_baker_or_no_read_barriers = !isR6 && (!kUseReadBarrier || kUseBakerReadBarrier); + const bool baker_or_no_read_barriers = (!kUseReadBarrier || kUseBakerReadBarrier); InvokeRuntimeCallingConvention calling_convention; __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); // For HLoadString/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. Register entry_address = kNoRegister; - if (r2_baker_or_no_read_barriers) { + if (baker_or_no_read_barriers) { Register temp = locations->GetTemp(0).AsRegister<Register>(); bool temp_is_a0 = (temp == calling_convention.GetRegisterAt(0)); // In the unlucky case that `temp` is A0, we preserve the address in `out` across the @@ -339,9 +355,18 @@ class LoadStringSlowPathMIPS : public SlowPathCodeMIPS { CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); // Store the resolved string to the BSS entry. - if (r2_baker_or_no_read_barriers) { + if (baker_or_no_read_barriers) { // The string entry address was preserved in `entry_address` thanks to kSaveEverything. - __ StoreToOffset(kStoreWord, calling_convention.GetRegisterAt(0), entry_address, 0); + DCHECK(bss_info_high_); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + mips_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index, bss_info_high_); + bool reordering = __ SetReorder(false); + __ Bind(&info_low->label); + __ StoreToOffset(kStoreWord, + calling_convention.GetRegisterAt(0), + entry_address, + /* placeholder */ 0x5678); + __ SetReorder(reordering); } Primitive::Type type = instruction_->GetType(); @@ -351,14 +376,17 @@ class LoadStringSlowPathMIPS : public SlowPathCodeMIPS { RestoreLiveRegisters(codegen, locations); // Store the resolved string to the BSS entry. - if (!r2_baker_or_no_read_barriers) { - // For non-Baker read barriers (or on R6), we need to re-calculate the address of + if (!baker_or_no_read_barriers) { + // For non-Baker read barriers we need to re-calculate the address of // the string entry. + const bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); Register base = isR6 ? ZERO : locations->InAt(0).AsRegister<Register>(); - CodeGeneratorMIPS::PcRelativePatchInfo* info = + CodeGeneratorMIPS::PcRelativePatchInfo* info_high = mips_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + mips_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index, info_high); bool reordering = __ SetReorder(false); - mips_codegen->EmitPcRelativeAddressPlaceholderHigh(info, TMP, base); + mips_codegen->EmitPcRelativeAddressPlaceholderHigh(info_high, TMP, base, info_low); __ StoreToOffset(kStoreWord, out, TMP, /* placeholder */ 0x5678); __ SetReorder(reordering); } @@ -368,6 +396,9 @@ class LoadStringSlowPathMIPS : public SlowPathCodeMIPS { const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathMIPS"; } private: + // Pointer to the high half PC-relative patch info. + const CodeGeneratorMIPS::PcRelativePatchInfo* bss_info_high_; + DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathMIPS); }; @@ -403,10 +434,13 @@ class SuspendCheckSlowPathMIPS : public SlowPathCodeMIPS { : SlowPathCodeMIPS(instruction), successor_(successor) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); // Only saves live vector registers for SIMD. mips_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); CheckEntrypointTypes<kQuickTestSuspend, void, void>(); + RestoreLiveRegisters(codegen, locations); // Only restores live vector registers for SIMD. if (successor_ == nullptr) { __ B(GetReturnLabel()); } else { @@ -622,7 +656,7 @@ class ReadBarrierMarkSlowPathMIPS : public SlowPathCodeMIPS { __ NopIfNoReordering(); } else { int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -716,7 +750,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathMIPS : public SlowPathCodeMIPS { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -954,7 +988,9 @@ class ReadBarrierForHeapReferenceSlowPathMIPS : public SlowPathCodeMIPS { this); CheckEntrypointTypes< kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); - mips_codegen->Move32(out_, calling_convention.GetReturnLocation(Primitive::kPrimNot)); + mips_codegen->MoveLocation(out_, + calling_convention.GetReturnLocation(Primitive::kPrimNot), + Primitive::kPrimNot); RestoreLiveRegisters(codegen, locations); __ B(GetExitLabel()); @@ -1017,13 +1053,17 @@ class ReadBarrierForRootSlowPathMIPS : public SlowPathCodeMIPS { InvokeRuntimeCallingConvention calling_convention; CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); - mips_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); + mips_codegen->MoveLocation(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + root_, + Primitive::kPrimNot); mips_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, instruction_, instruction_->GetDexPc(), this); CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); - mips_codegen->Move32(out_, calling_convention.GetReturnLocation(Primitive::kPrimNot)); + mips_codegen->MoveLocation(out_, + calling_convention.GetReturnLocation(Primitive::kPrimNot), + Primitive::kPrimNot); RestoreLiveRegisters(codegen, locations); __ B(GetExitLabel()); @@ -1060,8 +1100,8 @@ CodeGeneratorMIPS::CodeGeneratorMIPS(HGraph* graph, isa_features_(isa_features), uint32_literals_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1411,106 +1451,112 @@ void CodeGeneratorMIPS::Bind(HBasicBlock* block) { __ Bind(GetLabelOf(block)); } -void CodeGeneratorMIPS::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { - if (src.Equals(dst)) { - return; - } - - if (src.IsConstant()) { - MoveConstant(dst, src.GetConstant()); - } else { - if (Primitive::Is64BitType(dst_type)) { - Move64(dst, src); - } else { - Move32(dst, src); - } - } +VectorRegister VectorRegisterFrom(Location location) { + DCHECK(location.IsFpuRegister()); + return static_cast<VectorRegister>(location.AsFpuRegister<FRegister>()); } -void CodeGeneratorMIPS::Move32(Location destination, Location source) { +void CodeGeneratorMIPS::MoveLocation(Location destination, + Location source, + Primitive::Type dst_type) { if (source.Equals(destination)) { return; } - if (destination.IsRegister()) { - if (source.IsRegister()) { - __ Move(destination.AsRegister<Register>(), source.AsRegister<Register>()); - } else if (source.IsFpuRegister()) { - __ Mfc1(destination.AsRegister<Register>(), source.AsFpuRegister<FRegister>()); - } else { - DCHECK(source.IsStackSlot()) << "Cannot move from " << source << " to " << destination; - __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); - } - } else if (destination.IsFpuRegister()) { - if (source.IsRegister()) { - __ Mtc1(source.AsRegister<Register>(), destination.AsFpuRegister<FRegister>()); - } else if (source.IsFpuRegister()) { - __ MovS(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); - } else { - DCHECK(source.IsStackSlot()) << "Cannot move from " << source << " to " << destination; - __ LoadSFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); - } + if (source.IsConstant()) { + MoveConstant(destination, source.GetConstant()); } else { - DCHECK(destination.IsStackSlot()) << destination; - if (source.IsRegister()) { - __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); - } else if (source.IsFpuRegister()) { - __ StoreSToOffset(source.AsFpuRegister<FRegister>(), SP, destination.GetStackIndex()); - } else { - DCHECK(source.IsStackSlot()) << "Cannot move from " << source << " to " << destination; - __ LoadFromOffset(kLoadWord, TMP, SP, source.GetStackIndex()); - __ StoreToOffset(kStoreWord, TMP, SP, destination.GetStackIndex()); - } - } -} - -void CodeGeneratorMIPS::Move64(Location destination, Location source) { - if (source.Equals(destination)) { - return; - } - - if (destination.IsRegisterPair()) { - if (source.IsRegisterPair()) { - __ Move(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); - __ Move(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); - } else if (source.IsFpuRegister()) { - Register dst_high = destination.AsRegisterPairHigh<Register>(); - Register dst_low = destination.AsRegisterPairLow<Register>(); - FRegister src = source.AsFpuRegister<FRegister>(); - __ Mfc1(dst_low, src); - __ MoveFromFpuHigh(dst_high, src); - } else { - DCHECK(source.IsDoubleStackSlot()) << "Cannot move from " << source << " to " << destination; - int32_t off = source.GetStackIndex(); - Register r = destination.AsRegisterPairLow<Register>(); - __ LoadFromOffset(kLoadDoubleword, r, SP, off); - } - } else if (destination.IsFpuRegister()) { - if (source.IsRegisterPair()) { - FRegister dst = destination.AsFpuRegister<FRegister>(); - Register src_high = source.AsRegisterPairHigh<Register>(); - Register src_low = source.AsRegisterPairLow<Register>(); - __ Mtc1(src_low, dst); - __ MoveToFpuHigh(src_high, dst); - } else if (source.IsFpuRegister()) { - __ MovD(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); - } else { - DCHECK(source.IsDoubleStackSlot()) << "Cannot move from " << source << " to " << destination; - __ LoadDFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); - } - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - int32_t off = destination.GetStackIndex(); - if (source.IsRegisterPair()) { - __ StoreToOffset(kStoreDoubleword, source.AsRegisterPairLow<Register>(), SP, off); - } else if (source.IsFpuRegister()) { - __ StoreDToOffset(source.AsFpuRegister<FRegister>(), SP, off); + if (destination.IsRegister()) { + if (source.IsRegister()) { + __ Move(destination.AsRegister<Register>(), source.AsRegister<Register>()); + } else if (source.IsFpuRegister()) { + __ Mfc1(destination.AsRegister<Register>(), source.AsFpuRegister<FRegister>()); + } else { + DCHECK(source.IsStackSlot()) << "Cannot move from " << source << " to " << destination; + __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); + } + } else if (destination.IsRegisterPair()) { + if (source.IsRegisterPair()) { + __ Move(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); + __ Move(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); + } else if (source.IsFpuRegister()) { + Register dst_high = destination.AsRegisterPairHigh<Register>(); + Register dst_low = destination.AsRegisterPairLow<Register>(); + FRegister src = source.AsFpuRegister<FRegister>(); + __ Mfc1(dst_low, src); + __ MoveFromFpuHigh(dst_high, src); + } else { + DCHECK(source.IsDoubleStackSlot()) << "Cannot move from " << source << " to " << destination; + int32_t off = source.GetStackIndex(); + Register r = destination.AsRegisterPairLow<Register>(); + __ LoadFromOffset(kLoadDoubleword, r, SP, off); + } + } else if (destination.IsFpuRegister()) { + if (source.IsRegister()) { + DCHECK(!Primitive::Is64BitType(dst_type)); + __ Mtc1(source.AsRegister<Register>(), destination.AsFpuRegister<FRegister>()); + } else if (source.IsRegisterPair()) { + DCHECK(Primitive::Is64BitType(dst_type)); + FRegister dst = destination.AsFpuRegister<FRegister>(); + Register src_high = source.AsRegisterPairHigh<Register>(); + Register src_low = source.AsRegisterPairLow<Register>(); + __ Mtc1(src_low, dst); + __ MoveToFpuHigh(src_high, dst); + } else if (source.IsFpuRegister()) { + if (GetGraph()->HasSIMD()) { + __ MoveV(VectorRegisterFrom(destination), + VectorRegisterFrom(source)); + } else { + if (Primitive::Is64BitType(dst_type)) { + __ MovD(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); + } else { + DCHECK_EQ(dst_type, Primitive::kPrimFloat); + __ MovS(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); + } + } + } else if (source.IsSIMDStackSlot()) { + __ LoadQFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); + } else if (source.IsDoubleStackSlot()) { + DCHECK(Primitive::Is64BitType(dst_type)); + __ LoadDFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); + } else { + DCHECK(!Primitive::Is64BitType(dst_type)); + DCHECK(source.IsStackSlot()) << "Cannot move from " << source << " to " << destination; + __ LoadSFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); + } + } else if (destination.IsSIMDStackSlot()) { + if (source.IsFpuRegister()) { + __ StoreQToOffset(source.AsFpuRegister<FRegister>(), SP, destination.GetStackIndex()); + } else { + DCHECK(source.IsSIMDStackSlot()); + __ LoadQFromOffset(FTMP, SP, source.GetStackIndex()); + __ StoreQToOffset(FTMP, SP, destination.GetStackIndex()); + } + } else if (destination.IsDoubleStackSlot()) { + int32_t dst_offset = destination.GetStackIndex(); + if (source.IsRegisterPair()) { + __ StoreToOffset(kStoreDoubleword, source.AsRegisterPairLow<Register>(), SP, dst_offset); + } else if (source.IsFpuRegister()) { + __ StoreDToOffset(source.AsFpuRegister<FRegister>(), SP, dst_offset); + } else { + DCHECK(source.IsDoubleStackSlot()) << "Cannot move from " << source << " to " << destination; + __ LoadFromOffset(kLoadWord, TMP, SP, source.GetStackIndex()); + __ StoreToOffset(kStoreWord, TMP, SP, dst_offset); + __ LoadFromOffset(kLoadWord, TMP, SP, source.GetStackIndex() + 4); + __ StoreToOffset(kStoreWord, TMP, SP, dst_offset + 4); + } } else { - DCHECK(source.IsDoubleStackSlot()) << "Cannot move from " << source << " to " << destination; - __ LoadFromOffset(kLoadWord, TMP, SP, source.GetStackIndex()); - __ StoreToOffset(kStoreWord, TMP, SP, off); - __ LoadFromOffset(kLoadWord, TMP, SP, source.GetStackIndex() + 4); - __ StoreToOffset(kStoreWord, TMP, SP, off + 4); + DCHECK(destination.IsStackSlot()) << destination; + int32_t dst_offset = destination.GetStackIndex(); + if (source.IsRegister()) { + __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, dst_offset); + } else if (source.IsFpuRegister()) { + __ StoreSToOffset(source.AsFpuRegister<FRegister>(), SP, dst_offset); + } else { + DCHECK(source.IsStackSlot()) << "Cannot move from " << source << " to " << destination; + __ LoadFromOffset(kLoadWord, TMP, SP, source.GetStackIndex()); + __ StoreToOffset(kStoreWord, TMP, SP, dst_offset); + } } } } @@ -1588,28 +1634,27 @@ inline void CodeGeneratorMIPS::EmitPcRelativeLinkerPatches( for (const PcRelativePatchInfo& info : infos) { const DexFile& dex_file = info.target_dex_file; size_t offset_or_index = info.offset_or_index; - DCHECK(info.high_label.IsBound()); - uint32_t high_offset = __ GetLabelLocation(&info.high_label); + DCHECK(info.label.IsBound()); + uint32_t literal_offset = __ GetLabelLocation(&info.label); // On R2 we use HMipsComputeBaseMethodAddress and patch relative to // the assembler's base label used for PC-relative addressing. - uint32_t pc_rel_offset = info.pc_rel_label.IsBound() - ? __ GetLabelLocation(&info.pc_rel_label) + const PcRelativePatchInfo& info_high = info.patch_info_high ? *info.patch_info_high : info; + uint32_t pc_rel_offset = info_high.pc_rel_label.IsBound() + ? __ GetLabelLocation(&info_high.pc_rel_label) : __ GetPcRelBaseLabelLocation(); - linker_patches->push_back(Factory(high_offset, &dex_file, pc_rel_offset, offset_or_index)); + linker_patches->push_back(Factory(literal_offset, &dex_file, pc_rel_offset, offset_or_index)); } } void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + pc_relative_method_patches_.size() + + method_bss_entry_patches_.size() + pc_relative_type_patches_.size() + type_bss_entry_patches_.size() + pc_relative_string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -1623,41 +1668,58 @@ void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patch EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); } CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeMethodPatch( - MethodReference target_method) { + MethodReference target_method, + const PcRelativePatchInfo* info_high) { return NewPcRelativePatch(*target_method.dex_file, target_method.dex_method_index, + info_high, &pc_relative_method_patches_); } +CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewMethodBssEntryPatch( + MethodReference target_method, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + info_high, + &method_bss_entry_patches_); +} + CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeTypePatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); + const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(dex_file, type_index.index_, info_high, &pc_relative_type_patches_); } CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewTypeBssEntryPatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); + const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(dex_file, type_index.index_, info_high, &type_bss_entry_patches_); } CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeStringPatch( - const DexFile& dex_file, dex::StringIndex string_index) { - return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); -} - -CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); + const DexFile& dex_file, + dex::StringIndex string_index, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(dex_file, string_index.index_, info_high, &pc_relative_string_patches_); } CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativePatch( - const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { - patches->emplace_back(dex_file, offset_or_index); + const DexFile& dex_file, + uint32_t offset_or_index, + const PcRelativePatchInfo* info_high, + ArenaDeque<PcRelativePatchInfo>* patches) { + patches->emplace_back(dex_file, offset_or_index, info_high); return &patches->back(); } @@ -1671,14 +1733,16 @@ Literal* CodeGeneratorMIPS::DeduplicateBootImageAddressLiteral(uint32_t address) return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); } -void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, +void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info_high, Register out, - Register base) { + Register base, + PcRelativePatchInfo* info_low) { + DCHECK(!info_high->patch_info_high); DCHECK_NE(out, base); if (GetInstructionSetFeatures().IsR6()) { DCHECK_EQ(base, ZERO); - __ Bind(&info->high_label); - __ Bind(&info->pc_rel_label); + __ Bind(&info_high->label); + __ Bind(&info_high->pc_rel_label); // Add the high half of a 32-bit offset to PC. __ Auipc(out, /* placeholder */ 0x1234); } else { @@ -1687,18 +1751,20 @@ void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo // Generate a dummy PC-relative call to obtain PC. __ Nal(); } - __ Bind(&info->high_label); + __ Bind(&info_high->label); __ Lui(out, /* placeholder */ 0x1234); // If we emitted the NAL, bind the pc_rel_label, otherwise base is a register holding // the HMipsComputeBaseMethodAddress which has its own label stored in MipsAssembler. if (base == ZERO) { - __ Bind(&info->pc_rel_label); + __ Bind(&info_high->pc_rel_label); } // Add the high half of a 32-bit offset to PC. __ Addu(out, out, (base == ZERO) ? RA : base); } - // The immediately following instruction will add the sign-extended low half of the 32-bit + // A following instruction will add the sign-extended low half of the 32-bit // offset to `out` (e.g. lw, jialc, addiu). + DCHECK_EQ(info_low->patch_info_high, info_high); + __ Bind(&info_low->label); } CodeGeneratorMIPS::JitPatchInfo* CodeGeneratorMIPS::NewJitRootStringPatch( @@ -1832,13 +1898,21 @@ size_t CodeGeneratorMIPS::RestoreCoreRegister(size_t stack_index, uint32_t reg_i } size_t CodeGeneratorMIPS::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ StoreDToOffset(FRegister(reg_id), SP, stack_index); - return kMipsDoublewordSize; + if (GetGraph()->HasSIMD()) { + __ StoreQToOffset(FRegister(reg_id), SP, stack_index); + } else { + __ StoreDToOffset(FRegister(reg_id), SP, stack_index); + } + return GetFloatingPointSpillSlotSize(); } size_t CodeGeneratorMIPS::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ LoadDFromOffset(FRegister(reg_id), SP, stack_index); - return kMipsDoublewordSize; + if (GetGraph()->HasSIMD()) { + __ LoadQFromOffset(FRegister(reg_id), SP, stack_index); + } else { + __ LoadDFromOffset(FRegister(reg_id), SP, stack_index); + } + return GetFloatingPointSpillSlotSize(); } void CodeGeneratorMIPS::DumpCoreRegister(std::ostream& stream, int reg) const { @@ -2287,7 +2361,7 @@ void InstructionCodeGeneratorMIPS::HandleShift(HBinaryOperation* instr) { Register lhs_low = locations->InAt(0).AsRegisterPairLow<Register>(); if (use_imm) { if (shift_value == 0) { - codegen_->Move64(locations->Out(), locations->InAt(0)); + codegen_->MoveLocation(locations->Out(), locations->InAt(0), type); } else if (shift_value < kMipsBitsPerWord) { if (has_ins_rotr) { if (instr->IsShl()) { @@ -6423,7 +6497,7 @@ void InstructionCodeGeneratorMIPS::GenerateGcRootFieldLoad(HInstruction* instruc // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(root.reg() - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(root.reg() - 1); // Loading the entrypoint does not require a load acquire since it is only changed when // threads are suspended or running a checkpoint. __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); @@ -7000,7 +7074,7 @@ HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. - // TODO: Create as many MipsDexCacheArraysBase instructions as needed for methods + // TODO: Create as many HMipsComputeBaseMethodAddress instructions as needed for methods // with irreducible loops. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); bool is_r6 = GetInstructionSetFeatures().IsR6(); @@ -7030,6 +7104,8 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many HMipsComputeBaseMethodAddress instructions as needed for methods + // with irreducible loops. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); bool is_r6 = GetInstructionSetFeatures().IsR6(); bool fallback_load = has_irreducible_loops && !is_r6; @@ -7093,25 +7169,28 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticO HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many HMipsComputeBaseMethodAddress instructions as needed for methods + // with irreducible loops. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); bool is_r6 = GetInstructionSetFeatures().IsR6(); bool fallback_load = has_irreducible_loops && !is_r6; switch (dispatch_info.method_load_kind) { case HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative: - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: break; default: fallback_load = false; break; } if (fallback_load) { - dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; + dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall; dispatch_info.method_load_data = 0; } return dispatch_info; } -void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorMIPS::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // All registers are assumed to be correctly set up per the calling convention. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. HInvokeStaticOrDirect::MethodLoadKind method_load_kind = invoke->GetMethodLoadKind(); @@ -7137,10 +7216,12 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke break; case HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative: { DCHECK(GetCompilerOptions().IsBootImage()); - PcRelativePatchInfo* info = NewPcRelativeMethodPatch(invoke->GetTargetMethod()); + PcRelativePatchInfo* info_high = NewPcRelativeMethodPatch(invoke->GetTargetMethod()); + PcRelativePatchInfo* info_low = + NewPcRelativeMethodPatch(invoke->GetTargetMethod(), info_high); bool reordering = __ SetReorder(false); Register temp_reg = temp.AsRegister<Register>(); - EmitPcRelativeAddressPlaceholderHigh(info, TMP, base_reg); + EmitPcRelativeAddressPlaceholderHigh(info_high, TMP, base_reg, info_low); __ Addiu(temp_reg, TMP, /* placeholder */ 0x5678); __ SetReorder(reordering); break; @@ -7148,51 +7229,22 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ LoadConst32(temp.AsRegister<Register>(), invoke->GetMethodAddress()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - if (is_r6) { - uint32_t offset = invoke->GetDexCacheArrayOffset(); - CodeGeneratorMIPS::PcRelativePatchInfo* info = - NewPcRelativeDexCacheArrayPatch(invoke->GetDexFileForPcRelativeDexCache(), offset); - bool reordering = __ SetReorder(false); - EmitPcRelativeAddressPlaceholderHigh(info, TMP, ZERO); - __ Lw(temp.AsRegister<Register>(), TMP, /* placeholder */ 0x5678); - __ SetReorder(reordering); - } else { - HMipsDexCacheArraysBase* base = - invoke->InputAt(invoke->GetSpecialInputIndex())->AsMipsDexCacheArraysBase(); - int32_t offset = - invoke->GetDexCacheArrayOffset() - base->GetElementOffset() - kDexCacheArrayLwOffset; - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); - } - break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register reg = temp.AsRegister<Register>(); - Register method_reg; - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - // TODO: use the appropriate DCHECK() here if possible. - // DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ Lw(reg, SP, kCurrentMethodStackOffset); - } - - // temp = temp->dex_cache_resolved_methods_; - __ LoadFromOffset(kLoadWord, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kMipsPointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ LoadFromOffset(kLoadWord, - reg, - reg, - CodeGenerator::GetCachePointerOffset(index_in_cache)); + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { + PcRelativePatchInfo* info_high = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); + PcRelativePatchInfo* info_low = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()), info_high); + Register temp_reg = temp.AsRegister<Register>(); + bool reordering = __ SetReorder(false); + EmitPcRelativeAddressPlaceholderHigh(info_high, TMP, base_reg, info_low); + __ Lw(temp_reg, TMP, /* placeholder */ 0x5678); + __ SetReorder(reordering); break; } + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. + } } switch (code_ptr_location) { @@ -7211,6 +7263,8 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke __ NopIfNoReordering(); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + DCHECK(!IsLeafMethod()); } @@ -7228,10 +7282,10 @@ void InstructionCodeGeneratorMIPS::VisitInvokeStaticOrDirect(HInvokeStaticOrDire locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } -void CodeGeneratorMIPS::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { +void CodeGeneratorMIPS::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { // Use the calling convention instead of the location of the receiver, as // intrinsics may have put the receiver in a different register. In the intrinsics // slow path, the arguments have been moved to the right place, so here we are @@ -7263,6 +7317,7 @@ void CodeGeneratorMIPS::GenerateVirtualCall(HInvokeVirtual* invoke, Location tem // T9(); __ Jalr(T9); __ NopIfNoReordering(); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void InstructionCodeGeneratorMIPS::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -7272,7 +7327,6 @@ void InstructionCodeGeneratorMIPS::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderMIPS::VisitLoadClass(HLoadClass* cls) { @@ -7312,11 +7366,8 @@ void LocationsBuilderMIPS::VisitLoadClass(HLoadClass* cls) { if (load_kind == HLoadClass::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { // Rely on the type resolution or initialization and marking to save everything we need. - // Request a temp to hold the BSS entry location for the slow path on R2 - // (no benefit for R6). - if (!isR6) { - locations->AddTemp(Location::RequiresRegister()); - } + // Request a temp to hold the BSS entry location for the slow path. + locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); @@ -7362,6 +7413,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF ? kWithoutReadBarrier : kCompilerReadBarrierOption; bool generate_null_check = false; + CodeGeneratorMIPS::PcRelativePatchInfo* bss_info_high = nullptr; switch (load_kind) { case HLoadClass::LoadKind::kReferrersClass: { DCHECK(!cls->CanCallRuntime()); @@ -7377,10 +7429,15 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { DCHECK(codegen_->GetCompilerOptions().IsBootImage()); DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); - CodeGeneratorMIPS::PcRelativePatchInfo* info = + CodeGeneratorMIPS::PcRelativePatchInfo* info_high = codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex(), info_high); bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, + out, + base_or_current_method_reg, + info_low); __ Addiu(out, out, /* placeholder */ 0x5678); __ SetReorder(reordering); break; @@ -7396,24 +7453,18 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF break; } case HLoadClass::LoadKind::kBssEntry: { - CodeGeneratorMIPS::PcRelativePatchInfo* info = - codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); + bss_info_high = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex(), bss_info_high); constexpr bool non_baker_read_barrier = kUseReadBarrier && !kUseBakerReadBarrier; - if (isR6 || non_baker_read_barrier) { - bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); - GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678, read_barrier_option); - __ SetReorder(reordering); - } else { - // On R2 save the BSS entry address in a temporary register instead of - // recalculating it in the slow path. - Register temp = locations->GetTemp(0).AsRegister<Register>(); - bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, temp, base_or_current_method_reg); - __ Addiu(temp, temp, /* placeholder */ 0x5678); - __ SetReorder(reordering); - GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); - } + Register temp = non_baker_read_barrier ? out : locations->GetTemp(0).AsRegister<Register>(); + bool reordering = __ SetReorder(false); + codegen_->EmitPcRelativeAddressPlaceholderHigh(bss_info_high, + temp, + base_or_current_method_reg, + info_low); + GenerateGcRootFieldLoad(cls, out_loc, temp, /* placeholder */ 0x5678, read_barrier_option); + __ SetReorder(reordering); generate_null_check = true; break; } @@ -7437,7 +7488,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF if (generate_null_check || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathMIPS( - cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); + cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck(), bss_info_high); codegen_->AddSlowPath(slow_path); if (generate_null_check) { __ Beqz(out, slow_path->GetEntryLabel()); @@ -7502,11 +7553,8 @@ void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { // Rely on the pResolveString and marking to save everything we need. - // Request a temp to hold the BSS entry location for the slow path on R2 - // (no benefit for R6). - if (!isR6) { - locations->AddTemp(Location::RequiresRegister()); - } + // Request a temp to hold the BSS entry location for the slow path. + locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); @@ -7542,10 +7590,15 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ switch (load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { DCHECK(codegen_->GetCompilerOptions().IsBootImage()); - CodeGeneratorMIPS::PcRelativePatchInfo* info = + CodeGeneratorMIPS::PcRelativePatchInfo* info_high = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex(), info_high); bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, + out, + base_or_current_method_reg, + info_low); __ Addiu(out, out, /* placeholder */ 0x5678); __ SetReorder(reordering); return; // No dex cache slow path. @@ -7561,29 +7614,25 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ } case HLoadString::LoadKind::kBssEntry: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - CodeGeneratorMIPS::PcRelativePatchInfo* info = + CodeGeneratorMIPS::PcRelativePatchInfo* info_high = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); + CodeGeneratorMIPS::PcRelativePatchInfo* info_low = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex(), info_high); constexpr bool non_baker_read_barrier = kUseReadBarrier && !kUseBakerReadBarrier; - if (isR6 || non_baker_read_barrier) { - bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); - GenerateGcRootFieldLoad(load, - out_loc, - out, - /* placeholder */ 0x5678, - kCompilerReadBarrierOption); - __ SetReorder(reordering); - } else { - // On R2 save the BSS entry address in a temporary register instead of - // recalculating it in the slow path. - Register temp = locations->GetTemp(0).AsRegister<Register>(); - bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, temp, base_or_current_method_reg); - __ Addiu(temp, temp, /* placeholder */ 0x5678); - __ SetReorder(reordering); - GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); - } - SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathMIPS(load); + Register temp = non_baker_read_barrier ? out : locations->GetTemp(0).AsRegister<Register>(); + bool reordering = __ SetReorder(false); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, + temp, + base_or_current_method_reg, + info_low); + GenerateGcRootFieldLoad(load, + out_loc, + temp, + /* placeholder */ 0x5678, + kCompilerReadBarrierOption); + __ SetReorder(reordering); + SlowPathCodeMIPS* slow_path = + new (GetGraph()->GetArena()) LoadStringSlowPathMIPS(load, info_high); codegen_->AddSlowPath(slow_path); __ Beqz(out, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); @@ -7810,8 +7859,11 @@ void LocationsBuilderMIPS::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorMIPS::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes care // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); + DCHECK(!codegen_->IsLeafMethod()); } void LocationsBuilderMIPS::VisitNewInstance(HNewInstance* instruction) { @@ -8198,7 +8250,11 @@ void InstructionCodeGeneratorMIPS::VisitUnresolvedStaticFieldSet( void LocationsBuilderMIPS::VisitSuspendCheck(HSuspendCheck* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + // In suspend check slow path, usually there are no caller-save registers at all. + // If SIMD instructions are present, however, we force spilling all live SIMD + // registers in full width (since the runtime only saves/restores lower part). + locations->SetCustomSlowPathCallerSaves( + GetGraph()->HasSIMD() ? RegisterSet::AllFpu() : RegisterSet::Empty()); } void InstructionCodeGeneratorMIPS::VisitSuspendCheck(HSuspendCheck* instruction) { @@ -8732,29 +8788,11 @@ void InstructionCodeGeneratorMIPS::VisitMipsComputeBaseMethodAddress( __ Nal(); // Grab the return address off RA. __ Move(reg, RA); - // TODO: Can we share this code with that of VisitMipsDexCacheArraysBase()? // Remember this offset (the obtained PC value) for later use with constant area. __ BindPcRelBaseLabel(); } -void LocationsBuilderMIPS::VisitMipsDexCacheArraysBase(HMipsDexCacheArraysBase* base) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorMIPS::VisitMipsDexCacheArraysBase(HMipsDexCacheArraysBase* base) { - Register reg = base->GetLocations()->Out().AsRegister<Register>(); - CodeGeneratorMIPS::PcRelativePatchInfo* info = - codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); - CHECK(!codegen_->GetInstructionSetFeatures().IsR6()); - bool reordering = __ SetReorder(false); - // TODO: Reuse MipsComputeBaseMethodAddress on R2 instead of passing ZERO to force emitting NAL. - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, reg, ZERO); - __ Addiu(reg, reg, /* placeholder */ 0x5678); - __ SetReorder(reordering); -} - void LocationsBuilderMIPS::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { // The trampoline uses the same calling convention as dex calling conventions, // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index 736b5070d9..52ee852269 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -61,6 +61,8 @@ static constexpr FRegister kFpuCalleeSaves[] = class CodeGeneratorMIPS; +VectorRegister VectorRegisterFrom(Location location); + class InvokeDexCallingConvention : public CallingConvention<Register, FRegister> { public: InvokeDexCallingConvention() @@ -344,6 +346,10 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { uint32_t num_entries, HBasicBlock* switch_block, HBasicBlock* default_block); + + int32_t VecAddress(LocationSummary* locations, + size_t size, + /* out */ Register* adjusted_base); void GenConditionalMoveR2(HSelect* select); void GenConditionalMoveR6(HSelect* select); @@ -368,13 +374,15 @@ class CodeGeneratorMIPS : public CodeGenerator { void Bind(HBasicBlock* block) OVERRIDE; - void Move32(Location destination, Location source); - void Move64(Location destination, Location source); void MoveConstant(Location location, HConstant* c); size_t GetWordSize() const OVERRIDE { return kMipsWordSize; } - size_t GetFloatingPointSpillSlotSize() const OVERRIDE { return kMipsDoublewordSize; } + size_t GetFloatingPointSpillSlotSize() const OVERRIDE { + return GetGraph()->HasSIMD() + ? 2 * kMipsDoublewordSize // 16 bytes for each spill. + : 1 * kMipsDoublewordSize; // 8 bytes for each spill. + } uintptr_t GetAddressOf(HBasicBlock* block) OVERRIDE { return assembler_.GetLabelLocation(GetLabelOf(block)); @@ -552,8 +560,10 @@ class CodeGeneratorMIPS : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp); - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED, Primitive::Type type ATTRIBUTE_UNUSED) OVERRIDE { @@ -566,32 +576,68 @@ class CodeGeneratorMIPS : public CodeGenerator { // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays // and boot image strings. The only difference is the interpretation of the offset_or_index. + // The 16-bit halves of the 32-bit PC-relative offset are patched separately, necessitating + // two patches/infos. There can be more than two patches/infos if the instruction supplying + // the high half is shared with e.g. a slow path, while the low half is supplied by separate + // instructions, e.g.: + // lui r1, high // patch + // addu r1, r1, rbase + // lw r2, low(r1) // patch + // beqz r2, slow_path + // back: + // ... + // slow_path: + // ... + // sw r2, low(r1) // patch + // b back struct PcRelativePatchInfo { - PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) - : target_dex_file(dex_file), offset_or_index(off_or_idx) { } - PcRelativePatchInfo(PcRelativePatchInfo&& other) = default; + PcRelativePatchInfo(const DexFile& dex_file, + uint32_t off_or_idx, + const PcRelativePatchInfo* info_high) + : target_dex_file(dex_file), + offset_or_index(off_or_idx), + label(), + pc_rel_label(), + patch_info_high(info_high) { } const DexFile& target_dex_file; // Either the dex cache array element offset or the string/type index. uint32_t offset_or_index; - // Label for the instruction loading the most significant half of the offset that's added to PC - // to form the base address (the least significant half is loaded with the instruction that - // follows). - MipsLabel high_label; - // Label for the instruction corresponding to PC+0. + // Label for the instruction to patch. + MipsLabel label; + // Label for the instruction corresponding to PC+0. Not bound or used in low half patches. + // Not bound in high half patches on R2 when using HMipsComputeBaseMethodAddress. + // Bound in high half patches on R2 when using the NAL instruction instead of + // HMipsComputeBaseMethodAddress. + // Bound in high half patches on R6. MipsLabel pc_rel_label; + // Pointer to the info for the high half patch or nullptr if this is the high half patch info. + const PcRelativePatchInfo* patch_info_high; + + private: + PcRelativePatchInfo(PcRelativePatchInfo&& other) = delete; + DISALLOW_COPY_AND_ASSIGN(PcRelativePatchInfo); }; - PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); - PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); - PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); + PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method, + const PcRelativePatchInfo* info_high = nullptr); + PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method, + const PcRelativePatchInfo* info_high = nullptr); + PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high = nullptr); + PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high = nullptr); PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, - dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); + dex::StringIndex string_index, + const PcRelativePatchInfo* info_high = nullptr); Literal* DeduplicateBootImageAddressLiteral(uint32_t address); - void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, Register out, Register base); + void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info_high, + Register out, + Register base, + PcRelativePatchInfo* info_low); // The JitPatchInfo is used for JIT string and class loads. struct JitPatchInfo { @@ -626,6 +672,7 @@ class CodeGeneratorMIPS : public CodeGenerator { Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, uint32_t offset_or_index, + const PcRelativePatchInfo* info_high, ArenaDeque<PcRelativePatchInfo>* patches); template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> @@ -643,10 +690,10 @@ class CodeGeneratorMIPS : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // PC-relative patch info for each HMipsDexCacheArraysBase. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 6026814f04..454a2ddc14 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -164,19 +164,42 @@ class LoadClassSlowPathMIPS64 : public SlowPathCodeMIPS64 { LoadClassSlowPathMIPS64(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, - bool do_clinit) - : SlowPathCodeMIPS64(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { + bool do_clinit, + const CodeGeneratorMIPS64::PcRelativePatchInfo* bss_info_high = nullptr) + : SlowPathCodeMIPS64(at), + cls_(cls), + dex_pc_(dex_pc), + do_clinit_(do_clinit), + bss_info_high_(bss_info_high) { DCHECK(at->IsLoadClass() || at->IsClinitCheck()); } void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); - + const bool baker_or_no_read_barriers = (!kUseReadBarrier || kUseBakerReadBarrier); + InvokeRuntimeCallingConvention calling_convention; + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + const bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + GpuRegister entry_address = kNoGpuRegister; + if (is_load_class_bss_entry && baker_or_no_read_barriers) { + GpuRegister temp = locations->GetTemp(0).AsRegister<GpuRegister>(); + bool temp_is_a0 = (temp == calling_convention.GetRegisterAt(0)); + // In the unlucky case that `temp` is A0, we preserve the address in `out` across the + // kSaveEverything call. + entry_address = temp_is_a0 ? out.AsRegister<GpuRegister>() : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_a0) { + __ Move(entry_address, temp); + } + } + dex::TypeIndex type_index = cls_->GetTypeIndex(); __ LoadConst32(calling_convention.GetRegisterAt(0), type_index.index_); QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage @@ -188,8 +211,20 @@ class LoadClassSlowPathMIPS64 : public SlowPathCodeMIPS64 { CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); } + // For HLoadClass/kBssEntry, store the resolved class to the BSS entry. + if (is_load_class_bss_entry && baker_or_no_read_barriers) { + // The class entry address was preserved in `entry_address` thanks to kSaveEverything. + DCHECK(bss_info_high_); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + mips64_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index, bss_info_high_); + __ Bind(&info_low->label); + __ StoreToOffset(kStoreWord, + calling_convention.GetRegisterAt(0), + entry_address, + /* placeholder */ 0x5678); + } + // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); Primitive::Type type = instruction_->GetType(); @@ -197,16 +232,18 @@ class LoadClassSlowPathMIPS64 : public SlowPathCodeMIPS64 { Location::RegisterLocation(calling_convention.GetRegisterAt(0)), type); } - RestoreLiveRegisters(codegen, locations); - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { - DCHECK(out.IsValid()); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = + + // For HLoadClass/kBssEntry, store the resolved class to the BSS entry. + if (is_load_class_bss_entry && !baker_or_no_read_barriers) { + // For non-Baker read barriers we need to re-calculate the address of + // the class entry. + CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = mips64_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); - mips64_codegen->EmitPcRelativeAddressPlaceholderHigh(info, AT); - __ Sw(out.AsRegister<GpuRegister>(), AT, /* placeholder */ 0x5678); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + mips64_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index, info_high); + mips64_codegen->EmitPcRelativeAddressPlaceholderHigh(info_high, TMP, info_low); + __ StoreToOffset(kStoreWord, out.AsRegister<GpuRegister>(), TMP, /* placeholder */ 0x5678); } __ Bc(GetExitLabel()); } @@ -223,50 +260,94 @@ class LoadClassSlowPathMIPS64 : public SlowPathCodeMIPS64 { // Whether to initialize the class. const bool do_clinit_; + // Pointer to the high half PC-relative patch info for HLoadClass/kBssEntry. + const CodeGeneratorMIPS64::PcRelativePatchInfo* bss_info_high_; + DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathMIPS64); }; class LoadStringSlowPathMIPS64 : public SlowPathCodeMIPS64 { public: - explicit LoadStringSlowPathMIPS64(HLoadString* instruction) : SlowPathCodeMIPS64(instruction) {} + explicit LoadStringSlowPathMIPS64(HLoadString* instruction, + const CodeGeneratorMIPS64::PcRelativePatchInfo* bss_info_high) + : SlowPathCodeMIPS64(instruction), bss_info_high_(bss_info_high) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + DCHECK(instruction_->IsLoadString()); + DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); LocationSummary* locations = instruction_->GetLocations(); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); + HLoadString* load = instruction_->AsLoadString(); + const dex::StringIndex string_index = load->GetStringIndex(); + GpuRegister out = locations->Out().AsRegister<GpuRegister>(); CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); - + const bool baker_or_no_read_barriers = (!kUseReadBarrier || kUseBakerReadBarrier); + InvokeRuntimeCallingConvention calling_convention; __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; - HLoadString* load = instruction_->AsLoadString(); - const dex::StringIndex string_index = instruction_->AsLoadString()->GetStringIndex(); + // For HLoadString/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + GpuRegister entry_address = kNoGpuRegister; + if (baker_or_no_read_barriers) { + GpuRegister temp = locations->GetTemp(0).AsRegister<GpuRegister>(); + bool temp_is_a0 = (temp == calling_convention.GetRegisterAt(0)); + // In the unlucky case that `temp` is A0, we preserve the address in `out` across the + // kSaveEverything call. + entry_address = temp_is_a0 ? out : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_a0) { + __ Move(entry_address, temp); + } + } + __ LoadConst32(calling_convention.GetRegisterAt(0), string_index.index_); mips64_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); + + // Store the resolved string to the BSS entry. + if (baker_or_no_read_barriers) { + // The string entry address was preserved in `entry_address` thanks to kSaveEverything. + DCHECK(bss_info_high_); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + mips64_codegen->NewPcRelativeStringPatch(load->GetDexFile(), + string_index, + bss_info_high_); + __ Bind(&info_low->label); + __ StoreToOffset(kStoreWord, + calling_convention.GetRegisterAt(0), + entry_address, + /* placeholder */ 0x5678); + } + Primitive::Type type = instruction_->GetType(); mips64_codegen->MoveLocation(locations->Out(), Location::RegisterLocation(calling_convention.GetRegisterAt(0)), type); - RestoreLiveRegisters(codegen, locations); - // Store the resolved String to the BSS entry. - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = - mips64_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); - mips64_codegen->EmitPcRelativeAddressPlaceholderHigh(info, AT); - __ Sw(out, AT, /* placeholder */ 0x5678); - + // Store the resolved string to the BSS entry. + if (!baker_or_no_read_barriers) { + // For non-Baker read barriers we need to re-calculate the address of + // the string entry. + CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = + mips64_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + mips64_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index, info_high); + mips64_codegen->EmitPcRelativeAddressPlaceholderHigh(info_high, TMP, info_low); + __ StoreToOffset(kStoreWord, out, TMP, /* placeholder */ 0x5678); + } __ Bc(GetExitLabel()); } const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathMIPS64"; } private: + // Pointer to the high half PC-relative patch info. + const CodeGeneratorMIPS64::PcRelativePatchInfo* bss_info_high_; + DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathMIPS64); }; @@ -525,7 +606,7 @@ class ReadBarrierMarkSlowPathMIPS64 : public SlowPathCodeMIPS64 { __ Nop(); } else { int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -618,7 +699,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathMIPS64 : public SlowPathCodeMIPS64 { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -957,8 +1038,8 @@ CodeGeneratorMIPS64::CodeGeneratorMIPS64(HGraph* graph, graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1208,6 +1289,11 @@ void CodeGeneratorMIPS64::MoveLocation(Location destination, SP, source.GetStackIndex()); } + } else if (source.IsSIMDStackSlot()) { + __ LoadFpuFromOffset(kLoadQuadword, + destination.AsFpuRegister<FpuRegister>(), + SP, + source.GetStackIndex()); } else if (source.IsConstant()) { // Move to GPR/FPR from constant GpuRegister gpr = AT; @@ -1248,12 +1334,17 @@ void CodeGeneratorMIPS64::MoveLocation(Location destination, } } else if (source.IsFpuRegister()) { if (destination.IsFpuRegister()) { - // Move to FPR from FPR - if (dst_type == Primitive::kPrimFloat) { - __ MovS(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + if (GetGraph()->HasSIMD()) { + __ MoveV(VectorRegisterFrom(destination), + VectorRegisterFrom(source)); } else { - DCHECK_EQ(dst_type, Primitive::kPrimDouble); - __ MovD(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + // Move to FPR from FPR + if (dst_type == Primitive::kPrimFloat) { + __ MovS(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + } else { + DCHECK_EQ(dst_type, Primitive::kPrimDouble); + __ MovD(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + } } } else { DCHECK(destination.IsRegister()); @@ -1264,6 +1355,23 @@ void CodeGeneratorMIPS64::MoveLocation(Location destination, } } } + } else if (destination.IsSIMDStackSlot()) { + if (source.IsFpuRegister()) { + __ StoreFpuToOffset(kStoreQuadword, + source.AsFpuRegister<FpuRegister>(), + SP, + destination.GetStackIndex()); + } else { + DCHECK(source.IsSIMDStackSlot()); + __ LoadFpuFromOffset(kLoadQuadword, + FTMP, + SP, + source.GetStackIndex()); + __ StoreFpuToOffset(kStoreQuadword, + FTMP, + SP, + destination.GetStackIndex()); + } } else { // The destination is not a register. It must be a stack slot. DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot()); if (source.IsRegister() || source.IsFpuRegister()) { @@ -1431,23 +1539,23 @@ inline void CodeGeneratorMIPS64::EmitPcRelativeLinkerPatches( for (const PcRelativePatchInfo& info : infos) { const DexFile& dex_file = info.target_dex_file; size_t offset_or_index = info.offset_or_index; - DCHECK(info.pc_rel_label.IsBound()); - uint32_t pc_rel_offset = __ GetLabelLocation(&info.pc_rel_label); - linker_patches->push_back(Factory(pc_rel_offset, &dex_file, pc_rel_offset, offset_or_index)); + DCHECK(info.label.IsBound()); + uint32_t literal_offset = __ GetLabelLocation(&info.label); + const PcRelativePatchInfo& info_high = info.patch_info_high ? *info.patch_info_high : info; + uint32_t pc_rel_offset = __ GetLabelLocation(&info_high.label); + linker_patches->push_back(Factory(literal_offset, &dex_file, pc_rel_offset, offset_or_index)); } } void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + pc_relative_method_patches_.size() + + method_bss_entry_patches_.size() + pc_relative_type_patches_.size() + type_bss_entry_patches_.size() + pc_relative_string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -1461,41 +1569,58 @@ void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pat EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); } CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeMethodPatch( - MethodReference target_method) { + MethodReference target_method, + const PcRelativePatchInfo* info_high) { return NewPcRelativePatch(*target_method.dex_file, target_method.dex_method_index, + info_high, &pc_relative_method_patches_); } +CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewMethodBssEntryPatch( + MethodReference target_method, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + info_high, + &method_bss_entry_patches_); +} + CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeTypePatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); + const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(dex_file, type_index.index_, info_high, &pc_relative_type_patches_); } CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewTypeBssEntryPatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); + const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(dex_file, type_index.index_, info_high, &type_bss_entry_patches_); } CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeStringPatch( - const DexFile& dex_file, dex::StringIndex string_index) { - return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); -} - -CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); + const DexFile& dex_file, + dex::StringIndex string_index, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch(dex_file, string_index.index_, info_high, &pc_relative_string_patches_); } CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativePatch( - const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { - patches->emplace_back(dex_file, offset_or_index); + const DexFile& dex_file, + uint32_t offset_or_index, + const PcRelativePatchInfo* info_high, + ArenaDeque<PcRelativePatchInfo>* patches) { + patches->emplace_back(dex_file, offset_or_index, info_high); return &patches->back(); } @@ -1515,13 +1640,17 @@ Literal* CodeGeneratorMIPS64::DeduplicateBootImageAddressLiteral(uint64_t addres return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); } -void CodeGeneratorMIPS64::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, - GpuRegister out) { - __ Bind(&info->pc_rel_label); +void CodeGeneratorMIPS64::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info_high, + GpuRegister out, + PcRelativePatchInfo* info_low) { + DCHECK(!info_high->patch_info_high); + __ Bind(&info_high->label); // Add the high half of a 32-bit offset to PC. __ Auipc(out, /* placeholder */ 0x1234); - // The immediately following instruction will add the sign-extended low half of the 32-bit + // A following instruction will add the sign-extended low half of the 32-bit // offset to `out` (e.g. ld, jialc, daddiu). + DCHECK_EQ(info_low->patch_info_high, info_high); + __ Bind(&info_low->label); } Literal* CodeGeneratorMIPS64::DeduplicateJitStringLiteral(const DexFile& dex_file, @@ -4292,7 +4421,7 @@ void InstructionCodeGeneratorMIPS64::GenerateGcRootFieldLoad( // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(root.reg() - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(root.reg() - 1); // Loading the entrypoint does not require a load acquire since it is only changed when // threads are suspended or running a checkpoint. __ LoadFromOffset(kLoadDoubleword, temp.AsRegister<GpuRegister>(), TR, entry_point_offset); @@ -4915,7 +5044,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS64::GetSupportedInvokeStati return desired_dispatch_info; } -void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorMIPS64::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // All registers are assumed to be correctly set up per the calling convention. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. HInvokeStaticOrDirect::MethodLoadKind method_load_kind = invoke->GetMethodLoadKind(); @@ -4937,9 +5067,11 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo break; case HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative: { DCHECK(GetCompilerOptions().IsBootImage()); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = + CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = NewPcRelativeMethodPatch(invoke->GetTargetMethod()); - EmitPcRelativeAddressPlaceholderHigh(info, AT); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + NewPcRelativeMethodPatch(invoke->GetTargetMethod(), info_high); + EmitPcRelativeAddressPlaceholderHigh(info_high, AT, info_low); __ Daddiu(temp.AsRegister<GpuRegister>(), AT, /* placeholder */ 0x5678); break; } @@ -4948,41 +5080,18 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo kLoadDoubleword, DeduplicateUint64Literal(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { - uint32_t offset = invoke->GetDexCacheArrayOffset(); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = - NewPcRelativeDexCacheArrayPatch(invoke->GetDexFileForPcRelativeDexCache(), offset); - EmitPcRelativeAddressPlaceholderHigh(info, AT); + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { + PcRelativePatchInfo* info_high = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); + PcRelativePatchInfo* info_low = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()), info_high); + EmitPcRelativeAddressPlaceholderHigh(info_high, AT, info_low); __ Ld(temp.AsRegister<GpuRegister>(), AT, /* placeholder */ 0x5678); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - GpuRegister reg = temp.AsRegister<GpuRegister>(); - GpuRegister method_reg; - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<GpuRegister>(); - } else { - // TODO: use the appropriate DCHECK() here if possible. - // DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ Ld(reg, SP, kCurrentMethodStackOffset); - } - - // temp = temp->dex_cache_resolved_methods_; - __ LoadFromOffset(kLoadDoubleword, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kMips64PointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ LoadFromOffset(kLoadDoubleword, - reg, - reg, - CodeGenerator::GetCachePointerOffset(index_in_cache)); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } @@ -5002,6 +5111,8 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo __ Nop(); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + DCHECK(!IsLeafMethod()); } @@ -5019,10 +5130,10 @@ void InstructionCodeGeneratorMIPS64::VisitInvokeStaticOrDirect(HInvokeStaticOrDi locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } -void CodeGeneratorMIPS64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { +void CodeGeneratorMIPS64::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { // Use the calling convention instead of the location of the receiver, as // intrinsics may have put the receiver in a different register. In the intrinsics // slow path, the arguments have been moved to the right place, so here we are @@ -5054,6 +5165,7 @@ void CodeGeneratorMIPS64::GenerateVirtualCall(HInvokeVirtual* invoke, Location t // T9(); __ Jalr(T9); __ Nop(); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void InstructionCodeGeneratorMIPS64::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -5063,7 +5175,6 @@ void InstructionCodeGeneratorMIPS64::VisitInvokeVirtual(HInvokeVirtual* invoke) codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderMIPS64::VisitLoadClass(HLoadClass* cls) { @@ -5091,12 +5202,14 @@ void LocationsBuilderMIPS64::VisitLoadClass(HLoadClass* cls) { if (load_kind == HLoadClass::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { // Rely on the type resolution or initialization and marking to save everything we need. + // Request a temp to hold the BSS entry location for the slow path. + locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); locations->SetCustomSlowPathCallerSaves(caller_saves); } else { - // For non-Baker read barrier we have a temp-clobbering call. + // For non-Baker read barriers we have a temp-clobbering call. } } } @@ -5124,6 +5237,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S ? kWithoutReadBarrier : kCompilerReadBarrierOption; bool generate_null_check = false; + CodeGeneratorMIPS64::PcRelativePatchInfo* bss_info_high = nullptr; switch (load_kind) { case HLoadClass::LoadKind::kReferrersClass: DCHECK(!cls->CanCallRuntime()); @@ -5138,9 +5252,11 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { DCHECK(codegen_->GetCompilerOptions().IsBootImage()); DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = + CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, AT); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex(), info_high); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, AT, info_low); __ Daddiu(out, AT, /* placeholder */ 0x5678); break; } @@ -5155,10 +5271,15 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S break; } case HLoadClass::LoadKind::kBssEntry: { - CodeGeneratorMIPS64::PcRelativePatchInfo* info = - codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out); - GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678, read_barrier_option); + bss_info_high = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex(), bss_info_high); + constexpr bool non_baker_read_barrier = kUseReadBarrier && !kUseBakerReadBarrier; + GpuRegister temp = non_baker_read_barrier + ? out + : locations->GetTemp(0).AsRegister<GpuRegister>(); + codegen_->EmitPcRelativeAddressPlaceholderHigh(bss_info_high, temp, info_low); + GenerateGcRootFieldLoad(cls, out_loc, temp, /* placeholder */ 0x5678, read_barrier_option); generate_null_check = true; break; } @@ -5179,7 +5300,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S if (generate_null_check || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathMIPS64( - cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); + cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck(), bss_info_high); codegen_->AddSlowPath(slow_path); if (generate_null_check) { __ Beqzc(out, slow_path->GetEntryLabel()); @@ -5227,12 +5348,14 @@ void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) { if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { // Rely on the pResolveString and marking to save everything we need. + // Request a temp to hold the BSS entry location for the slow path. + locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); locations->SetCustomSlowPathCallerSaves(caller_saves); } else { - // For non-Baker read barrier we have a temp-clobbering call. + // For non-Baker read barriers we have a temp-clobbering call. } } } @@ -5249,9 +5372,11 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA switch (load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { DCHECK(codegen_->GetCompilerOptions().IsBootImage()); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = + CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, AT); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex(), info_high); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, AT, info_low); __ Daddiu(out, AT, /* placeholder */ 0x5678); return; // No dex cache slow path. } @@ -5266,15 +5391,22 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA } case HLoadString::LoadKind::kBssEntry: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = + CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out); + CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex(), info_high); + constexpr bool non_baker_read_barrier = kUseReadBarrier && !kUseBakerReadBarrier; + GpuRegister temp = non_baker_read_barrier + ? out + : locations->GetTemp(0).AsRegister<GpuRegister>(); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, temp, info_low); GenerateGcRootFieldLoad(load, out_loc, - out, + temp, /* placeholder */ 0x5678, kCompilerReadBarrierOption); - SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathMIPS64(load); + SlowPathCodeMIPS64* slow_path = + new (GetGraph()->GetArena()) LoadStringSlowPathMIPS64(load, info_high); codegen_->AddSlowPath(slow_path); __ Beqzc(out, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); @@ -5446,8 +5578,11 @@ void LocationsBuilderMIPS64::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorMIPS64::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes care // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); + DCHECK(!codegen_->IsLeafMethod()); } void LocationsBuilderMIPS64::VisitNewInstance(HNewInstance* instruction) { diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 9c6b6f62cb..c94cc93dad 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -59,6 +59,8 @@ static constexpr FpuRegister kFpuCalleeSaves[] = class CodeGeneratorMIPS64; +VectorRegister VectorRegisterFrom(Location location); + class InvokeDexCallingConvention : public CallingConvention<GpuRegister, FpuRegister> { public: InvokeDexCallingConvention() @@ -521,8 +523,10 @@ class CodeGeneratorMIPS64 : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED, Primitive::Type type ATTRIBUTE_UNUSED) OVERRIDE { @@ -536,30 +540,59 @@ class CodeGeneratorMIPS64 : public CodeGenerator { // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays, // boot image strings and method calls. The only difference is the interpretation of // the offset_or_index. + // The 16-bit halves of the 32-bit PC-relative offset are patched separately, necessitating + // two patches/infos. There can be more than two patches/infos if the instruction supplying + // the high half is shared with e.g. a slow path, while the low half is supplied by separate + // instructions, e.g.: + // auipc r1, high // patch + // lwu r2, low(r1) // patch + // beqzc r2, slow_path + // back: + // ... + // slow_path: + // ... + // sw r2, low(r1) // patch + // bc back struct PcRelativePatchInfo { - PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) - : target_dex_file(dex_file), offset_or_index(off_or_idx) { } - PcRelativePatchInfo(PcRelativePatchInfo&& other) = default; + PcRelativePatchInfo(const DexFile& dex_file, + uint32_t off_or_idx, + const PcRelativePatchInfo* info_high) + : target_dex_file(dex_file), + offset_or_index(off_or_idx), + label(), + patch_info_high(info_high) { } const DexFile& target_dex_file; // Either the dex cache array element offset or the string/type/method index. uint32_t offset_or_index; - // Label for the auipc instruction. - Mips64Label pc_rel_label; + // Label for the instruction to patch. + Mips64Label label; + // Pointer to the info for the high half patch or nullptr if this is the high half patch info. + const PcRelativePatchInfo* patch_info_high; + + private: + PcRelativePatchInfo(PcRelativePatchInfo&& other) = delete; + DISALLOW_COPY_AND_ASSIGN(PcRelativePatchInfo); }; - PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); - PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); - PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); + PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method, + const PcRelativePatchInfo* info_high = nullptr); + PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method, + const PcRelativePatchInfo* info_high = nullptr); + PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high = nullptr); + PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, + dex::TypeIndex type_index, + const PcRelativePatchInfo* info_high = nullptr); PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, - dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); - PcRelativePatchInfo* NewPcRelativeCallPatch(const DexFile& dex_file, - uint32_t method_index); + dex::StringIndex string_index, + const PcRelativePatchInfo* info_high = nullptr); Literal* DeduplicateBootImageAddressLiteral(uint64_t address); - void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, GpuRegister out); + void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info_high, + GpuRegister out, + PcRelativePatchInfo* info_low); void PatchJitRootUse(uint8_t* code, const uint8_t* roots_data, @@ -587,6 +620,7 @@ class CodeGeneratorMIPS64 : public CodeGenerator { PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, uint32_t offset_or_index, + const PcRelativePatchInfo* info_high, ArenaDeque<PcRelativePatchInfo>* patches); template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> @@ -607,10 +641,10 @@ class CodeGeneratorMIPS64 : public CodeGenerator { // Deduplication map for 64-bit literals, used for non-patchable method address or method code // address. Uint64ToLiteralMap uint64_literals_; - // PC-relative patch info. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_vector_arm.cc b/compiler/optimizing/code_generator_vector_arm.cc deleted file mode 100644 index f8552dcfc9..0000000000 --- a/compiler/optimizing/code_generator_vector_arm.cc +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "code_generator_arm.h" - -namespace art { -namespace arm { - -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT - -void LocationsBuilderARM::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecSetScalars(HVecSetScalars* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecSetScalars(HVecSetScalars* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecSumReduce(HVecSumReduce* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecSumReduce(HVecSumReduce* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -// Helper to set up locations for vector unary operations. -static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { - LocationSummary* locations = new (arena) LocationSummary(instruction); - switch (instruction->GetPackedType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - DCHECK(locations); - break; - default: - LOG(FATAL) << "Unsupported SIMD type"; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitVecCnv(HVecCnv* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecCnv(HVecCnv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecNeg(HVecNeg* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecNeg(HVecNeg* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecAbs(HVecAbs* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAbs(HVecAbs* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecNot(HVecNot* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecNot(HVecNot* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -// Helper to set up locations for vector binary operations. -static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { - LocationSummary* locations = new (arena) LocationSummary(instruction); - switch (instruction->GetPackedType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - DCHECK(locations); - break; - default: - LOG(FATAL) << "Unsupported SIMD type"; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitVecAdd(HVecAdd* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAdd(HVecAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecSub(HVecSub* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecSub(HVecSub* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMul(HVecMul* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecMul(HVecMul* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecDiv(HVecDiv* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecDiv(HVecDiv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMin(HVecMin* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecMin(HVecMin* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMax(HVecMax* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecMax(HVecMax* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecAnd(HVecAnd* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAnd(HVecAnd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecAndNot(HVecAndNot* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAndNot(HVecAndNot* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecOr(HVecOr* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecOr(HVecOr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecXor(HVecXor* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecXor(HVecXor* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -// Helper to set up locations for vector shift operations. -static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { - LocationSummary* locations = new (arena) LocationSummary(instruction); - switch (instruction->GetPackedType()) { - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimLong: - DCHECK(locations); - break; - default: - LOG(FATAL) << "Unsupported SIMD type"; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitVecShl(HVecShl* instruction) { - CreateVecShiftLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecShl(HVecShl* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecShr(HVecShr* instruction) { - CreateVecShiftLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecShr(HVecShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecUShr(HVecUShr* instruction) { - CreateVecShiftLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecUShr(HVecUShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) { - LOG(FATAL) << "No SIMD for " << instr->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) { - LOG(FATAL) << "No SIMD for " << instr->GetId(); -} - -void LocationsBuilderARM::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -#undef __ - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/code_generator_vector_arm64.cc b/compiler/optimizing/code_generator_vector_arm64.cc index a41adca02c..f422b9fc8b 100644 --- a/compiler/optimizing/code_generator_vector_arm64.cc +++ b/compiler/optimizing/code_generator_vector_arm64.cc @@ -22,6 +22,8 @@ using namespace vixl::aarch64; // NOLINT(build/namespaces) namespace art { namespace arm64 { +using helpers::ARM64EncodableConstantOrRegister; +using helpers::Arm64CanEncodeConstantAsImmediate; using helpers::DRegisterFrom; using helpers::VRegisterFrom; using helpers::HeapOperand; @@ -34,6 +36,7 @@ using helpers::WRegisterFrom; void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + HInstruction* input = instruction->InputAt(0); switch (instruction->GetPackedType()) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: @@ -41,13 +44,19 @@ void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruc case Primitive::kPrimShort: case Primitive::kPrimInt: case Primitive::kPrimLong: - locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(0, ARM64EncodableConstantOrRegister(input, instruction)); locations->SetOut(Location::RequiresFpuRegister()); break; case Primitive::kPrimFloat: case Primitive::kPrimDouble: - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + if (input->IsConstant() && + Arm64CanEncodeConstantAsImmediate(input->AsConstant(), instruction)) { + locations->SetInAt(0, Location::ConstantLocation(input->AsConstant())); + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + } break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -57,33 +66,58 @@ void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruc void InstructionCodeGeneratorARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { LocationSummary* locations = instruction->GetLocations(); + Location src_loc = locations->InAt(0); VRegister dst = VRegisterFrom(locations->Out()); switch (instruction->GetPackedType()) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: DCHECK_EQ(16u, instruction->GetVectorLength()); - __ Dup(dst.V16B(), InputRegisterAt(instruction, 0)); + if (src_loc.IsConstant()) { + __ Movi(dst.V16B(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V16B(), InputRegisterAt(instruction, 0)); + } break; case Primitive::kPrimChar: case Primitive::kPrimShort: DCHECK_EQ(8u, instruction->GetVectorLength()); - __ Dup(dst.V8H(), InputRegisterAt(instruction, 0)); + if (src_loc.IsConstant()) { + __ Movi(dst.V8H(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V8H(), InputRegisterAt(instruction, 0)); + } break; case Primitive::kPrimInt: DCHECK_EQ(4u, instruction->GetVectorLength()); - __ Dup(dst.V4S(), InputRegisterAt(instruction, 0)); + if (src_loc.IsConstant()) { + __ Movi(dst.V4S(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V4S(), InputRegisterAt(instruction, 0)); + } break; case Primitive::kPrimLong: DCHECK_EQ(2u, instruction->GetVectorLength()); - __ Dup(dst.V2D(), XRegisterFrom(locations->InAt(0))); + if (src_loc.IsConstant()) { + __ Movi(dst.V2D(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V2D(), XRegisterFrom(src_loc)); + } break; case Primitive::kPrimFloat: DCHECK_EQ(4u, instruction->GetVectorLength()); - __ Dup(dst.V4S(), VRegisterFrom(locations->InAt(0)).V4S(), 0); + if (src_loc.IsConstant()) { + __ Fmov(dst.V4S(), src_loc.GetConstant()->AsFloatConstant()->GetValue()); + } else { + __ Dup(dst.V4S(), VRegisterFrom(src_loc).V4S(), 0); + } break; case Primitive::kPrimDouble: DCHECK_EQ(2u, instruction->GetVectorLength()); - __ Dup(dst.V2D(), VRegisterFrom(locations->InAt(0)).V2D(), 0); + if (src_loc.IsConstant()) { + __ Fmov(dst.V2D(), src_loc.GetConstant()->AsDoubleConstant()->GetValue()); + } else { + __ Dup(dst.V2D(), VRegisterFrom(src_loc).V2D(), 0); + } break; default: LOG(FATAL) << "Unsupported SIMD type"; diff --git a/compiler/optimizing/code_generator_vector_arm_vixl.cc b/compiler/optimizing/code_generator_vector_arm_vixl.cc index 53f314ec40..527691d9d9 100644 --- a/compiler/optimizing/code_generator_vector_arm_vixl.cc +++ b/compiler/optimizing/code_generator_vector_arm_vixl.cc @@ -15,19 +15,62 @@ */ #include "code_generator_arm_vixl.h" +#include "mirror/array-inl.h" + +namespace vixl32 = vixl::aarch32; +using namespace vixl32; // NOLINT(build/namespaces) namespace art { namespace arm { -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ reinterpret_cast<ArmVIXLAssembler*>(GetAssembler())->GetVIXLAssembler()-> // NOLINT +using helpers::DRegisterFrom; +using helpers::Int64ConstantFrom; +using helpers::InputDRegisterAt; +using helpers::InputRegisterAt; +using helpers::OutputDRegister; +using helpers::RegisterFrom; + +#define __ GetVIXLAssembler()-> void LocationsBuilderARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void InstructionCodeGeneratorARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vdup(Untyped8, dst, InputRegisterAt(instruction, 0)); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vdup(Untyped16, dst, InputRegisterAt(instruction, 0)); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vdup(Untyped32, dst, InputRegisterAt(instruction, 0)); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) { @@ -51,13 +94,17 @@ static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* in LocationSummary* locations = new (arena) LocationSummary(instruction); switch (instruction->GetPackedType()) { case Primitive::kPrimBoolean: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), + instruction->IsVecNot() ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + break; case Primitive::kPrimByte: case Primitive::kPrimChar: case Primitive::kPrimShort: case Primitive::kPrimInt: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -78,7 +125,27 @@ void LocationsBuilderARMVIXL::VisitVecNeg(HVecNeg* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecNeg(HVecNeg* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister src = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vneg(DataTypeValue::S8, dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vneg(DataTypeValue::S16, dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vneg(DataTypeValue::S32, dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecAbs(HVecAbs* instruction) { @@ -86,7 +153,27 @@ void LocationsBuilderARMVIXL::VisitVecAbs(HVecAbs* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecAbs(HVecAbs* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister src = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vabs(DataTypeValue::S8, dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vabs(DataTypeValue::S16, dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vabs(DataTypeValue::S32, dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecNot(HVecNot* instruction) { @@ -94,7 +181,25 @@ void LocationsBuilderARMVIXL::VisitVecNot(HVecNot* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecNot(HVecNot* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister src = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: // special case boolean-not + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vmov(I8, dst, 1); + __ Veor(dst, dst, src); + break; + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + __ Vmvn(I8, dst, src); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } // Helper to set up locations for vector binary operations. @@ -106,9 +211,9 @@ static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* case Primitive::kPrimChar: case Primitive::kPrimShort: case Primitive::kPrimInt: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -121,7 +226,28 @@ void LocationsBuilderARMVIXL::VisitVecAdd(HVecAdd* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecAdd(HVecAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vadd(I8, dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vadd(I16, dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vadd(I32, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { @@ -129,7 +255,40 @@ void LocationsBuilderARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + instruction->IsRounded() + ? __ Vrhadd(DataTypeValue::U8, dst, lhs, rhs) + : __ Vhadd(DataTypeValue::U8, dst, lhs, rhs); + } else { + instruction->IsRounded() + ? __ Vrhadd(DataTypeValue::S8, dst, lhs, rhs) + : __ Vhadd(DataTypeValue::S8, dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + instruction->IsRounded() + ? __ Vrhadd(DataTypeValue::U16, dst, lhs, rhs) + : __ Vhadd(DataTypeValue::U16, dst, lhs, rhs); + } else { + instruction->IsRounded() + ? __ Vrhadd(DataTypeValue::S16, dst, lhs, rhs) + : __ Vhadd(DataTypeValue::S16, dst, lhs, rhs); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecSub(HVecSub* instruction) { @@ -137,7 +296,28 @@ void LocationsBuilderARMVIXL::VisitVecSub(HVecSub* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecSub(HVecSub* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vsub(I8, dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vsub(I16, dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vsub(I32, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecMul(HVecMul* instruction) { @@ -145,7 +325,28 @@ void LocationsBuilderARMVIXL::VisitVecMul(HVecMul* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecMul(HVecMul* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vmul(I8, dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vmul(I16, dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vmul(I32, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecDiv(HVecDiv* instruction) { @@ -161,7 +362,40 @@ void LocationsBuilderARMVIXL::VisitVecMin(HVecMin* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecMin(HVecMin* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Vmin(DataTypeValue::U8, dst, lhs, rhs); + } else { + __ Vmin(DataTypeValue::S8, dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Vmin(DataTypeValue::U16, dst, lhs, rhs); + } else { + __ Vmin(DataTypeValue::S16, dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Vmin(DataTypeValue::U32, dst, lhs, rhs); + } else { + __ Vmin(DataTypeValue::S32, dst, lhs, rhs); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecMax(HVecMax* instruction) { @@ -169,7 +403,40 @@ void LocationsBuilderARMVIXL::VisitVecMax(HVecMax* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecMax(HVecMax* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Vmax(DataTypeValue::U8, dst, lhs, rhs); + } else { + __ Vmax(DataTypeValue::S8, dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Vmax(DataTypeValue::U16, dst, lhs, rhs); + } else { + __ Vmax(DataTypeValue::S16, dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Vmax(DataTypeValue::U32, dst, lhs, rhs); + } else { + __ Vmax(DataTypeValue::S32, dst, lhs, rhs); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecAnd(HVecAnd* instruction) { @@ -177,7 +444,22 @@ void LocationsBuilderARMVIXL::VisitVecAnd(HVecAnd* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecAnd(HVecAnd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + __ Vand(I8, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecAndNot(HVecAndNot* instruction) { @@ -193,7 +475,22 @@ void LocationsBuilderARMVIXL::VisitVecOr(HVecOr* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecOr(HVecOr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + __ Vorr(I8, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecXor(HVecXor* instruction) { @@ -201,7 +498,22 @@ void LocationsBuilderARMVIXL::VisitVecXor(HVecXor* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecXor(HVecXor* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + __ Veor(I8, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } // Helper to set up locations for vector shift operations. @@ -212,8 +524,9 @@ static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* case Primitive::kPrimChar: case Primitive::kPrimShort: case Primitive::kPrimInt: - case Primitive::kPrimLong: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -226,7 +539,28 @@ void LocationsBuilderARMVIXL::VisitVecShl(HVecShl* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecShl(HVecShl* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vshl(I8, dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vshl(I16, dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vshl(I32, dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecShr(HVecShr* instruction) { @@ -234,7 +568,28 @@ void LocationsBuilderARMVIXL::VisitVecShr(HVecShr* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecShr(HVecShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vshr(DataTypeValue::S8, dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vshr(DataTypeValue::S16, dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vshr(DataTypeValue::S32, dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecUShr(HVecUShr* instruction) { @@ -242,7 +597,28 @@ void LocationsBuilderARMVIXL::VisitVecUShr(HVecUShr* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitVecUShr(HVecUShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vshr(DataTypeValue::U8, dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vshr(DataTypeValue::U16, dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Vshr(DataTypeValue::U32, dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) { @@ -253,20 +629,187 @@ void InstructionCodeGeneratorARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAcc LOG(FATAL) << "No SIMD for " << instr->GetId(); } +// Return whether the vector memory access operation is guaranteed to be word-aligned (ARM word +// size equals to 4). +static bool IsWordAligned(HVecMemoryOperation* instruction) { + return instruction->GetAlignment().IsAlignedAt(4u); +} + +// Helper to set up locations for vector memory operations. +static void CreateVecMemLocations(ArenaAllocator* arena, + HVecMemoryOperation* instruction, + bool is_load) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (is_load) { + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(2, Location::RequiresFpuRegister()); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector memory operations. Returns the memory operand and, +// if used, sets the output parameter scratch to a temporary register used in this operand, +// so that the client can release it right after the memory operand use. +MemOperand InstructionCodeGeneratorARMVIXL::VecAddress( + HVecMemoryOperation* instruction, + UseScratchRegisterScope* temps_scope, + /*out*/ vixl32::Register* scratch) { + LocationSummary* locations = instruction->GetLocations(); + vixl32::Register base = InputRegisterAt(instruction, 0); + + Location index = locations->InAt(1); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + uint32_t offset = mirror::Array::DataOffset(size).Uint32Value(); + size_t shift = ComponentSizeShiftWidth(size); + + // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet. + DCHECK(!instruction->InputAt(0)->IsIntermediateAddress()); + + if (index.IsConstant()) { + offset += Int64ConstantFrom(index) << shift; + return MemOperand(base, offset); + } else { + *scratch = temps_scope->Acquire(); + __ Add(*scratch, base, Operand(RegisterFrom(index), ShiftType::LSL, shift)); + + return MemOperand(*scratch, offset); + } +} + +AlignedMemOperand InstructionCodeGeneratorARMVIXL::VecAddressUnaligned( + HVecMemoryOperation* instruction, + UseScratchRegisterScope* temps_scope, + /*out*/ vixl32::Register* scratch) { + LocationSummary* locations = instruction->GetLocations(); + vixl32::Register base = InputRegisterAt(instruction, 0); + + Location index = locations->InAt(1); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + uint32_t offset = mirror::Array::DataOffset(size).Uint32Value(); + size_t shift = ComponentSizeShiftWidth(size); + + // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet. + DCHECK(!instruction->InputAt(0)->IsIntermediateAddress()); + + if (index.IsConstant()) { + offset += Int64ConstantFrom(index) << shift; + __ Add(*scratch, base, offset); + } else { + *scratch = temps_scope->Acquire(); + __ Add(*scratch, base, offset); + __ Add(*scratch, *scratch, Operand(RegisterFrom(index), ShiftType::LSL, shift)); + } + return AlignedMemOperand(*scratch, kNoAlignment); +} + void LocationsBuilderARMVIXL::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ true); } void InstructionCodeGeneratorARMVIXL::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + vixl32::DRegister reg = OutputDRegister(instruction); + UseScratchRegisterScope temps(GetVIXLAssembler()); + vixl32::Register scratch; + + DCHECK(instruction->GetPackedType() != Primitive::kPrimChar || !instruction->IsStringCharAt()); + + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (IsWordAligned(instruction)) { + __ Vldr(reg, VecAddress(instruction, &temps, &scratch)); + } else { + __ Vld1(Untyped8, + NeonRegisterList(reg, kMultipleLanes), + VecAddressUnaligned(instruction, &temps, &scratch)); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (IsWordAligned(instruction)) { + __ Vldr(reg, VecAddress(instruction, &temps, &scratch)); + } else { + __ Vld1(Untyped16, + NeonRegisterList(reg, kMultipleLanes), + VecAddressUnaligned(instruction, &temps, &scratch)); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (IsWordAligned(instruction)) { + __ Vldr(reg, VecAddress(instruction, &temps, &scratch)); + } else { + __ Vld1(Untyped32, + NeonRegisterList(reg, kMultipleLanes), + VecAddressUnaligned(instruction, &temps, &scratch)); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderARMVIXL::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ false); } void InstructionCodeGeneratorARMVIXL::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + vixl32::DRegister reg = InputDRegisterAt(instruction, 2); + UseScratchRegisterScope temps(GetVIXLAssembler()); + vixl32::Register scratch; + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (IsWordAligned(instruction)) { + __ Vstr(reg, VecAddress(instruction, &temps, &scratch)); + } else { + __ Vst1(Untyped8, + NeonRegisterList(reg, kMultipleLanes), + VecAddressUnaligned(instruction, &temps, &scratch)); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (IsWordAligned(instruction)) { + __ Vstr(reg, VecAddress(instruction, &temps, &scratch)); + } else { + __ Vst1(Untyped16, + NeonRegisterList(reg, kMultipleLanes), + VecAddressUnaligned(instruction, &temps, &scratch)); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (IsWordAligned(instruction)) { + __ Vstr(reg, VecAddress(instruction, &temps, &scratch)); + } else { + __ Vst1(Untyped32, + NeonRegisterList(reg, kMultipleLanes), + VecAddressUnaligned(instruction, &temps, &scratch)); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } #undef __ diff --git a/compiler/optimizing/code_generator_vector_mips.cc b/compiler/optimizing/code_generator_vector_mips.cc index c4a32252d9..ea36e90112 100644 --- a/compiler/optimizing/code_generator_vector_mips.cc +++ b/compiler/optimizing/code_generator_vector_mips.cc @@ -15,6 +15,7 @@ */ #include "code_generator_mips.h" +#include "mirror/array-inl.h" namespace art { namespace mips { @@ -23,11 +24,68 @@ namespace mips { #define __ down_cast<MipsAssembler*>(GetAssembler())-> // NOLINT void LocationsBuilderMIPS::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void InstructionCodeGeneratorMIPS::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ FillB(dst, locations->InAt(0).AsRegister<Register>()); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ FillH(dst, locations->InAt(0).AsRegister<Register>()); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, locations->InAt(0).AsRegister<Register>()); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Mtc1(locations->InAt(0).AsRegisterPairLow<Register>(), FTMP); + __ MoveToFpuHigh(locations->InAt(0).AsRegisterPairHigh<Register>(), FTMP); + __ ReplicateFPToVectorRegister(dst, FTMP, /* is_double */ true); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ ReplicateFPToVectorRegister(dst, + locations->InAt(0).AsFpuRegister<FRegister>(), + /* is_double */ false); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ ReplicateFPToVectorRegister(dst, + locations->InAt(0).AsFpuRegister<FRegister>(), + /* is_double */ true); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecSetScalars(HVecSetScalars* instruction) { @@ -51,13 +109,23 @@ static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* in LocationSummary* locations = new (arena) LocationSummary(instruction); switch (instruction->GetPackedType()) { case Primitive::kPrimBoolean: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), + instruction->IsVecNot() ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + break; case Primitive::kPrimByte: case Primitive::kPrimChar: case Primitive::kPrimShort: case Primitive::kPrimInt: + case Primitive::kPrimLong: case Primitive::kPrimFloat: case Primitive::kPrimDouble: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), + (instruction->IsVecNeg() || instruction->IsVecAbs()) + ? Location::kOutputOverlap + : Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -70,7 +138,17 @@ void LocationsBuilderMIPS::VisitVecCnv(HVecCnv* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecCnv(HVecCnv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + Primitive::Type from = instruction->GetInputType(); + Primitive::Type to = instruction->GetResultType(); + if (from == Primitive::kPrimInt && to == Primitive::kPrimFloat) { + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Ffint_sW(dst, src); + } else { + LOG(FATAL) << "Unsupported SIMD type"; + } } void LocationsBuilderMIPS::VisitVecNeg(HVecNeg* instruction) { @@ -78,7 +156,45 @@ void LocationsBuilderMIPS::VisitVecNeg(HVecNeg* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecNeg(HVecNeg* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ FillB(dst, ZERO); + __ SubvB(dst, dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ FillH(dst, ZERO); + __ SubvH(dst, dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ SubvW(dst, dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ SubvD(dst, dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ FsubW(dst, dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ FsubD(dst, dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecAbs(HVecAbs* instruction) { @@ -86,7 +202,47 @@ void LocationsBuilderMIPS::VisitVecAbs(HVecAbs* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecAbs(HVecAbs* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ FillB(dst, ZERO); // all zeroes + __ Add_aB(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ FillH(dst, ZERO); // all zeroes + __ Add_aH(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); // all zeroes + __ Add_aW(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); // all zeroes + __ Add_aD(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ LdiW(dst, -1); // all ones + __ SrliW(dst, dst, 1); + __ AndV(dst, dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ LdiD(dst, -1); // all ones + __ SrliD(dst, dst, 1); + __ AndV(dst, dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecNot(HVecNot* instruction) { @@ -94,7 +250,30 @@ void LocationsBuilderMIPS::VisitVecNot(HVecNot* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecNot(HVecNot* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: // special case boolean-not + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ LdiB(dst, 1); + __ XorV(dst, dst, src); + break; + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ NorV(dst, src, src); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } // Helper to set up locations for vector binary operations. @@ -106,9 +285,12 @@ static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* case Primitive::kPrimChar: case Primitive::kPrimShort: case Primitive::kPrimInt: + case Primitive::kPrimLong: case Primitive::kPrimFloat: case Primitive::kPrimDouble: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -121,7 +303,40 @@ void LocationsBuilderMIPS::VisitVecAdd(HVecAdd* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecAdd(HVecAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ AddvB(dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ AddvH(dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ AddvW(dst, lhs, rhs); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ AddvD(dst, lhs, rhs); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FaddW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FaddD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { @@ -129,7 +344,40 @@ void LocationsBuilderMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + instruction->IsRounded() + ? __ Aver_uB(dst, lhs, rhs) + : __ Ave_uB(dst, lhs, rhs); + } else { + instruction->IsRounded() + ? __ Aver_sB(dst, lhs, rhs) + : __ Ave_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + instruction->IsRounded() + ? __ Aver_uH(dst, lhs, rhs) + : __ Ave_uH(dst, lhs, rhs); + } else { + instruction->IsRounded() + ? __ Aver_sH(dst, lhs, rhs) + : __ Ave_sH(dst, lhs, rhs); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecSub(HVecSub* instruction) { @@ -137,7 +385,40 @@ void LocationsBuilderMIPS::VisitVecSub(HVecSub* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecSub(HVecSub* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SubvB(dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SubvH(dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SubvW(dst, lhs, rhs); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SubvD(dst, lhs, rhs); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FsubW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FsubD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMul(HVecMul* instruction) { @@ -145,7 +426,40 @@ void LocationsBuilderMIPS::VisitVecMul(HVecMul* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecMul(HVecMul* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ MulvB(dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ MulvH(dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ MulvW(dst, lhs, rhs); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ MulvD(dst, lhs, rhs); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FmulW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FmulD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecDiv(HVecDiv* instruction) { @@ -153,7 +467,23 @@ void LocationsBuilderMIPS::VisitVecDiv(HVecDiv* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecDiv(HVecDiv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FdivW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FdivD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMin(HVecMin* instruction) { @@ -161,7 +491,60 @@ void LocationsBuilderMIPS::VisitVecMin(HVecMin* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecMin(HVecMin* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uB(dst, lhs, rhs); + } else { + __ Min_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uH(dst, lhs, rhs); + } else { + __ Min_sH(dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uW(dst, lhs, rhs); + } else { + __ Min_sW(dst, lhs, rhs); + } + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uD(dst, lhs, rhs); + } else { + __ Min_sD(dst, lhs, rhs); + } + break; + // When one of arguments is NaN, fmin.df returns other argument, but Java expects a NaN value. + // TODO: Fix min(x, NaN) cases for float and double. + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FminW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FminD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMax(HVecMax* instruction) { @@ -169,7 +552,60 @@ void LocationsBuilderMIPS::VisitVecMax(HVecMax* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecMax(HVecMax* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uB(dst, lhs, rhs); + } else { + __ Max_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uH(dst, lhs, rhs); + } else { + __ Max_sH(dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uW(dst, lhs, rhs); + } else { + __ Max_sW(dst, lhs, rhs); + } + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uD(dst, lhs, rhs); + } else { + __ Max_sD(dst, lhs, rhs); + } + break; + // When one of arguments is NaN, fmax.df returns other argument, but Java expects a NaN value. + // TODO: Fix max(x, NaN) cases for float and double. + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FmaxW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FmaxD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecAnd(HVecAnd* instruction) { @@ -177,7 +613,27 @@ void LocationsBuilderMIPS::VisitVecAnd(HVecAnd* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecAnd(HVecAnd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ AndV(dst, lhs, rhs); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecAndNot(HVecAndNot* instruction) { @@ -193,7 +649,27 @@ void LocationsBuilderMIPS::VisitVecOr(HVecOr* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecOr(HVecOr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ OrV(dst, lhs, rhs); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecXor(HVecXor* instruction) { @@ -201,7 +677,27 @@ void LocationsBuilderMIPS::VisitVecXor(HVecXor* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecXor(HVecXor* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ XorV(dst, lhs, rhs); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } // Helper to set up locations for vector shift operations. @@ -213,7 +709,9 @@ static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* case Primitive::kPrimShort: case Primitive::kPrimInt: case Primitive::kPrimLong: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -226,7 +724,32 @@ void LocationsBuilderMIPS::VisitVecShl(HVecShl* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecShl(HVecShl* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SlliB(dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SlliH(dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SlliW(dst, lhs, value); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SlliD(dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecShr(HVecShr* instruction) { @@ -234,7 +757,32 @@ void LocationsBuilderMIPS::VisitVecShr(HVecShr* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecShr(HVecShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SraiB(dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SraiH(dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SraiW(dst, lhs, value); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SraiD(dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecUShr(HVecUShr* instruction) { @@ -242,7 +790,32 @@ void LocationsBuilderMIPS::VisitVecUShr(HVecUShr* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecUShr(HVecUShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SrliB(dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SrliH(dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SrliW(dst, lhs, value); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SrliD(dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) { @@ -253,20 +826,143 @@ void InstructionCodeGeneratorMIPS::VisitVecMultiplyAccumulate(HVecMultiplyAccumu LOG(FATAL) << "No SIMD for " << instr->GetId(); } +// Helper to set up locations for vector memory operations. +static void CreateVecMemLocations(ArenaAllocator* arena, + HVecMemoryOperation* instruction, + bool is_load) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (is_load) { + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(2, Location::RequiresFpuRegister()); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to prepare register and offset for vector memory operations. Returns the offset and sets +// the output parameter adjusted_base to the original base or to a reserved temporary register (AT). +int32_t InstructionCodeGeneratorMIPS::VecAddress(LocationSummary* locations, + size_t size, + /* out */ Register* adjusted_base) { + Register base = locations->InAt(0).AsRegister<Register>(); + Location index = locations->InAt(1); + int scale = TIMES_1; + switch (size) { + case 2: scale = TIMES_2; break; + case 4: scale = TIMES_4; break; + case 8: scale = TIMES_8; break; + default: break; + } + int32_t offset = mirror::Array::DataOffset(size).Int32Value(); + + if (index.IsConstant()) { + offset += index.GetConstant()->AsIntConstant()->GetValue() << scale; + __ AdjustBaseOffsetAndElementSizeShift(base, offset, scale); + *adjusted_base = base; + } else { + Register index_reg = index.AsRegister<Register>(); + if (scale != TIMES_1) { + __ Lsa(AT, index_reg, base, scale); + } else { + __ Addu(AT, base, index_reg); + } + *adjusted_base = AT; + } + return offset; +} + void LocationsBuilderMIPS::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /* is_load */ true); } void InstructionCodeGeneratorMIPS::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + VectorRegister reg = VectorRegisterFrom(locations->Out()); + Register base; + int32_t offset = VecAddress(locations, size, &base); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ LdB(reg, base, offset); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + // Loading 8-bytes (needed if dealing with compressed strings in StringCharAt) from unaligned + // memory address may cause a trap to the kernel if the CPU doesn't directly support unaligned + // loads and stores. + // TODO: Implement support for StringCharAt. + DCHECK(!instruction->IsStringCharAt()); + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ LdH(reg, base, offset); + break; + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ LdW(reg, base, offset); + break; + case Primitive::kPrimLong: + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ LdD(reg, base, offset); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /* is_load */ false); } void InstructionCodeGeneratorMIPS::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + VectorRegister reg = VectorRegisterFrom(locations->InAt(2)); + Register base; + int32_t offset = VecAddress(locations, size, &base); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ StB(reg, base, offset); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ StH(reg, base, offset); + break; + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ StW(reg, base, offset); + break; + case Primitive::kPrimLong: + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ StD(reg, base, offset); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } #undef __ diff --git a/compiler/optimizing/code_generator_vector_mips64.cc b/compiler/optimizing/code_generator_vector_mips64.cc index af9e89e791..0395db1df9 100644 --- a/compiler/optimizing/code_generator_vector_mips64.cc +++ b/compiler/optimizing/code_generator_vector_mips64.cc @@ -495,7 +495,60 @@ void LocationsBuilderMIPS64::VisitVecMin(HVecMin* instruction) { } void InstructionCodeGeneratorMIPS64::VisitVecMin(HVecMin* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uB(dst, lhs, rhs); + } else { + __ Min_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uH(dst, lhs, rhs); + } else { + __ Min_sH(dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uW(dst, lhs, rhs); + } else { + __ Min_sW(dst, lhs, rhs); + } + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uD(dst, lhs, rhs); + } else { + __ Min_sD(dst, lhs, rhs); + } + break; + // When one of arguments is NaN, fmin.df returns other argument, but Java expects a NaN value. + // TODO: Fix min(x, NaN) cases for float and double. + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FminW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FminD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS64::VisitVecMax(HVecMax* instruction) { @@ -503,7 +556,60 @@ void LocationsBuilderMIPS64::VisitVecMax(HVecMax* instruction) { } void InstructionCodeGeneratorMIPS64::VisitVecMax(HVecMax* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uB(dst, lhs, rhs); + } else { + __ Max_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uH(dst, lhs, rhs); + } else { + __ Max_sH(dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uW(dst, lhs, rhs); + } else { + __ Max_sW(dst, lhs, rhs); + } + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uD(dst, lhs, rhs); + } else { + __ Max_sD(dst, lhs, rhs); + } + break; + // When one of arguments is NaN, fmax.df returns other argument, but Java expects a NaN value. + // TODO: Fix max(x, NaN) cases for float and double. + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FmaxW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FmaxD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS64::VisitVecAnd(HVecAnd* instruction) { diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index b8465cd9d5..af0e6462a2 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -509,8 +509,7 @@ class ReadBarrierMarkSlowPathX86 : public SlowPathCode { // // rX <- ReadBarrierMarkRegX(rX) // - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ jmp(GetExitLabel()); @@ -595,8 +594,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathX86 : public SlowPathCode { // // rX <- ReadBarrierMarkRegX(rX) // - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); @@ -1032,8 +1030,8 @@ CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, move_resolver_(graph->GetArena(), this), assembler_(graph->GetArena()), isa_features_(isa_features), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1086,11 +1084,6 @@ void CodeGeneratorX86::GenerateFrameEntry() { } } - if (GetGraph()->HasShouldDeoptimizeFlag()) { - // Initialize should_deoptimize flag to 0. - __ movl(Address(ESP, -kShouldDeoptimizeFlagSize), Immediate(0)); - } - int adjust = GetFrameSize() - FrameEntrySpillSize(); __ subl(ESP, Immediate(adjust)); __ cfi().AdjustCFAOffset(adjust); @@ -1100,6 +1093,11 @@ void CodeGeneratorX86::GenerateFrameEntry() { if (RequiresCurrentMethod()) { __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument); } + + if (GetGraph()->HasShouldDeoptimizeFlag()) { + // Initialize should_deoptimize flag to 0. + __ movl(Address(ESP, GetStackOffsetOfShouldDeoptimizeFlag()), Immediate(0)); + } } void CodeGeneratorX86::GenerateFrameExit() { @@ -2204,7 +2202,6 @@ void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirec LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -2228,7 +2225,6 @@ void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) { @@ -4530,7 +4526,8 @@ Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOr return location.AsRegister<Register>(); } -void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorX86::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { @@ -4554,38 +4551,19 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, temp.AsRegister<Register>()); __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset)); // Bind a new fixup label at the end of the "movl" insn. - uint32_t offset = invoke->GetDexCacheArrayOffset(); - __ Bind(NewPcRelativeDexCacheArrayPatch( + __ Bind(NewMethodBssEntryPatch( invoke->InputAt(invoke->GetSpecialInputIndex())->AsX86ComputeBaseMethodAddress(), - invoke->GetDexFileForPcRelativeDexCache(), - offset)); + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()))); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register method_reg; - Register reg = temp.AsRegister<Register>(); - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ movl(reg, Address(ESP, kCurrentMethodStackOffset)); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ movl(reg, Address(method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value())); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache))); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } @@ -4600,11 +4578,13 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, kX86PointerSize).Int32Value())); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); DCHECK(!IsLeafMethod()); } -void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) { +void CodeGeneratorX86::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_in, SlowPathCode* slow_path) { Register temp = temp_in.AsRegister<Register>(); uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( invoke->GetVTableIndex(), kX86PointerSize).Uint32Value(); @@ -4632,6 +4612,7 @@ void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp // call temp->GetEntryPoint(); __ call(Address( temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86PointerSize).Int32Value())); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void CodeGeneratorX86::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { @@ -4644,6 +4625,16 @@ void CodeGeneratorX86::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { __ Bind(&boot_image_method_patches_.back().label); } +Label* CodeGeneratorX86::NewMethodBssEntryPatch( + HX86ComputeBaseMethodAddress* method_address, + MethodReference target_method) { + // Add the patch entry and bind its label at the end of the instruction. + method_bss_entry_patches_.emplace_back(method_address, + *target_method.dex_file, + target_method.dex_method_index); + return &method_bss_entry_patches_.back().label; +} + void CodeGeneratorX86::RecordBootTypePatch(HLoadClass* load_class) { HX86ComputeBaseMethodAddress* address = load_class->InputAt(0)->AsX86ComputeBaseMethodAddress(); boot_image_type_patches_.emplace_back(address, @@ -4678,15 +4669,6 @@ Label* CodeGeneratorX86::NewStringBssEntryPatch(HLoadString* load_string) { return &string_patches_.back().label; } -Label* CodeGeneratorX86::NewPcRelativeDexCacheArrayPatch( - HX86ComputeBaseMethodAddress* method_address, - const DexFile& dex_file, - uint32_t element_offset) { - // Add the patch entry and bind its label at the end of the instruction. - pc_relative_dex_cache_patches_.emplace_back(method_address, dex_file, element_offset); - return &pc_relative_dex_cache_patches_.back().label; -} - // The label points to the end of the "movl" or another instruction but the literal offset // for method patch needs to point to the embedded constant which occupies the last 4 bytes. constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u; @@ -4705,14 +4687,12 @@ inline void CodeGeneratorX86::EmitPcRelativeLinkerPatches( void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + boot_image_method_patches_.size() + + method_bss_entry_patches_.size() + boot_image_type_patches_.size() + type_bss_entry_patches_.size() + string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(boot_image_method_patches_, linker_patches); @@ -4724,6 +4704,8 @@ void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche DCHECK(boot_image_type_patches_.empty()); EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); @@ -7169,7 +7151,7 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad( // Test the entrypoint (`Thread::Current()->pReadBarrierMarkReg ## root.reg()`). const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(root.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(root.reg()); __ fs()->cmpl(Address::Absolute(entry_point_offset), Immediate(0)); // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 8130bd9d25..f48753b614 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -408,18 +408,19 @@ class CodeGeneratorX86 : public CodeGenerator { HInvokeStaticOrDirect* invoke) OVERRIDE; // Generate a call to a static or direct method. - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; // Generate a call to a virtual method. - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void RecordBootMethodPatch(HInvokeStaticOrDirect* invoke); + Label* NewMethodBssEntryPatch(HX86ComputeBaseMethodAddress* method_address, + MethodReference target_method); void RecordBootTypePatch(HLoadClass* load_class); Label* NewTypeBssEntryPatch(HLoadClass* load_class); void RecordBootStringPatch(HLoadString* load_string); Label* NewStringBssEntryPatch(HLoadString* load_string); - Label* NewPcRelativeDexCacheArrayPatch(HX86ComputeBaseMethodAddress* method_address, - const DexFile& dex_file, - uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index, Handle<mirror::String> handle); @@ -631,10 +632,10 @@ class CodeGeneratorX86 : public CodeGenerator { X86Assembler assembler_; const X86InstructionSetFeatures& isa_features_; - // PC-relative DexCache access info. - ArenaDeque<X86PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<X86PcRelativePatchInfo> boot_image_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<X86PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<X86PcRelativePatchInfo> boot_image_type_patches_; // Type patch locations for kBssEntry. diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 8dde298267..86f6d51734 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -524,7 +524,7 @@ class ReadBarrierMarkSlowPathX86_64 : public SlowPathCode { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); + Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ jmp(GetExitLabel()); @@ -615,7 +615,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathX86_64 : public SlowPathCode { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); + Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); @@ -977,8 +977,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86_64::GetSupportedInvokeStati return desired_dispatch_info; } -void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, - Location temp) { +void CodeGeneratorX86_64::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // All registers are assumed to be correctly set up. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. @@ -1002,35 +1002,17 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: Load64BitValue(temp.AsRegister<CpuRegister>(), invoke->GetMethodAddress()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { __ movq(temp.AsRegister<CpuRegister>(), Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); // Bind a new fixup label at the end of the "movl" insn. - uint32_t offset = invoke->GetDexCacheArrayOffset(); - __ Bind(NewPcRelativeDexCacheArrayPatch(invoke->GetDexFileForPcRelativeDexCache(), offset)); + __ Bind(NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()))); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register method_reg; - CpuRegister reg = temp.AsRegister<CpuRegister>(); - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg.AsRegister(); - __ movq(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset)); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ movq(reg, - Address(CpuRegister(method_reg), - ArtMethod::DexCacheResolvedMethodsOffset(kX86_64PointerSize).SizeValue())); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ movq(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache))); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } @@ -1045,11 +1027,13 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo kX86_64PointerSize).SizeValue())); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); DCHECK(!IsLeafMethod()); } -void CodeGeneratorX86_64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) { +void CodeGeneratorX86_64::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_in, SlowPathCode* slow_path) { CpuRegister temp = temp_in.AsRegister<CpuRegister>(); size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( invoke->GetVTableIndex(), kX86_64PointerSize).SizeValue(); @@ -1078,6 +1062,7 @@ void CodeGeneratorX86_64::GenerateVirtualCall(HInvokeVirtual* invoke, Location t // call temp->GetEntryPoint(); __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset( kX86_64PointerSize).SizeValue())); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void CodeGeneratorX86_64::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { @@ -1086,6 +1071,12 @@ void CodeGeneratorX86_64::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { __ Bind(&boot_image_method_patches_.back().label); } +Label* CodeGeneratorX86_64::NewMethodBssEntryPatch(MethodReference target_method) { + // Add a patch entry and return the label. + method_bss_entry_patches_.emplace_back(*target_method.dex_file, target_method.dex_method_index); + return &method_bss_entry_patches_.back().label; +} + void CodeGeneratorX86_64::RecordBootTypePatch(HLoadClass* load_class) { boot_image_type_patches_.emplace_back(load_class->GetDexFile(), load_class->GetTypeIndex().index_); @@ -1109,13 +1100,6 @@ Label* CodeGeneratorX86_64::NewStringBssEntryPatch(HLoadString* load_string) { return &string_patches_.back().label; } -Label* CodeGeneratorX86_64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset) { - // Add a patch entry and return the label. - pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset); - return &pc_relative_dex_cache_patches_.back().label; -} - // The label points to the end of the "movl" or another instruction but the literal offset // for method patch needs to point to the embedded constant which occupies the last 4 bytes. constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u; @@ -1134,14 +1118,12 @@ inline void CodeGeneratorX86_64::EmitPcRelativeLinkerPatches( void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + boot_image_method_patches_.size() + + method_bss_entry_patches_.size() + boot_image_type_patches_.size() + type_bss_entry_patches_.size() + string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(boot_image_method_patches_, linker_patches); @@ -1153,6 +1135,8 @@ void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pat DCHECK(boot_image_type_patches_.empty()); EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); @@ -1241,8 +1225,8 @@ CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph, assembler_(graph->GetArena()), isa_features_(isa_features), constant_area_start_(0), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1314,12 +1298,6 @@ void CodeGeneratorX86_64::GenerateFrameEntry() { } } - if (GetGraph()->HasShouldDeoptimizeFlag()) { - // Initialize should_deoptimize flag to 0. - __ movl(Address(CpuRegister(RSP), xmm_spill_location - kShouldDeoptimizeFlagSize), - Immediate(0)); - } - // Save the current method if we need it. Note that we do not // do this in HCurrentMethod, as the instruction might have been removed // in the SSA graph. @@ -1327,6 +1305,11 @@ void CodeGeneratorX86_64::GenerateFrameEntry() { __ movq(Address(CpuRegister(RSP), kCurrentMethodStackOffset), CpuRegister(kMethodRegisterArgument)); } + + if (GetGraph()->HasShouldDeoptimizeFlag()) { + // Initialize should_deoptimize flag to 0. + __ movl(Address(CpuRegister(RSP), GetStackOffsetOfShouldDeoptimizeFlag()), Immediate(0)); + } } void CodeGeneratorX86_64::GenerateFrameExit() { @@ -2387,7 +2370,6 @@ void InstructionCodeGeneratorX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDi LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86_64::HandleInvoke(HInvoke* invoke) { @@ -2411,7 +2393,6 @@ void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) { @@ -6559,7 +6540,7 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad( // Test the `Thread::Current()->pReadBarrierMarkReg ## root.reg()` entrypoint. const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(root.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(root.reg()); __ gs()->cmpl(Address::Absolute(entry_point_offset, /* no_rip */ true), Immediate(0)); // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 25479814d0..33c64290d4 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -404,15 +404,17 @@ class CodeGeneratorX86_64 : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void RecordBootMethodPatch(HInvokeStaticOrDirect* invoke); + Label* NewMethodBssEntryPatch(MethodReference target_method); void RecordBootTypePatch(HLoadClass* load_class); Label* NewTypeBssEntryPatch(HLoadClass* load_class); void RecordBootStringPatch(HLoadString* load_string); Label* NewStringBssEntryPatch(HLoadString* load_string); - Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index, Handle<mirror::String> handle); @@ -601,10 +603,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Used for fixups to the constant area. int constant_area_start_; - // PC-relative DexCache access info. - ArenaDeque<PatchInfo<Label>> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PatchInfo<Label>> boot_image_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PatchInfo<Label>> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PatchInfo<Label>> boot_image_type_patches_; // Type patch locations for kBssEntry. diff --git a/compiler/optimizing/codegen_test.cc b/compiler/optimizing/codegen_test.cc index fe25b7690d..0a8e97cf0d 100644 --- a/compiler/optimizing/codegen_test.cc +++ b/compiler/optimizing/codegen_test.cc @@ -43,8 +43,7 @@ static ::std::vector<CodegenTargetConfig> GetTargetConfigs() { ::std::vector<CodegenTargetConfig> v; ::std::vector<CodegenTargetConfig> test_config_candidates = { #ifdef ART_ENABLE_CODEGEN_arm - CodegenTargetConfig(kArm, create_codegen_arm), - CodegenTargetConfig(kThumb2, create_codegen_arm), + // TODO: Should't this be `kThumb2` instead of `kArm` here? CodegenTargetConfig(kArm, create_codegen_arm_vixl32), #endif #ifdef ART_ENABLE_CODEGEN_arm64 diff --git a/compiler/optimizing/codegen_test_utils.h b/compiler/optimizing/codegen_test_utils.h index 00a16fe849..1b38acd8b0 100644 --- a/compiler/optimizing/codegen_test_utils.h +++ b/compiler/optimizing/codegen_test_utils.h @@ -35,7 +35,6 @@ #include "ssa_liveness_analysis.h" #ifdef ART_ENABLE_CODEGEN_arm -#include "code_generator_arm.h" #include "code_generator_arm_vixl.h" #endif @@ -84,26 +83,6 @@ class CodegenTargetConfig { // in ART, and callee-save in C. Alternatively, we could use or write // the stub that saves and restores all registers, but it is easier // to just overwrite the code generator. -class TestCodeGeneratorARM : public arm::CodeGeneratorARM { - public: - TestCodeGeneratorARM(HGraph* graph, - const ArmInstructionSetFeatures& isa_features, - const CompilerOptions& compiler_options) - : arm::CodeGeneratorARM(graph, isa_features, compiler_options) { - AddAllocatedRegister(Location::RegisterLocation(arm::R6)); - AddAllocatedRegister(Location::RegisterLocation(arm::R7)); - } - - void SetupBlockedRegisters() const OVERRIDE { - arm::CodeGeneratorARM::SetupBlockedRegisters(); - blocked_core_registers_[arm::R4] = true; - blocked_core_registers_[arm::R6] = false; - blocked_core_registers_[arm::R7] = false; - } -}; - -// A way to test the VIXL32-based code generator on ARM. This will replace -// TestCodeGeneratorARM when the VIXL32-based backend replaces the existing one. class TestCodeGeneratorARMVIXL : public arm::CodeGeneratorARMVIXL { public: TestCodeGeneratorARMVIXL(HGraph* graph, @@ -288,14 +267,6 @@ static void RunCode(CodegenTargetConfig target_config, } #ifdef ART_ENABLE_CODEGEN_arm -CodeGenerator* create_codegen_arm(HGraph* graph, const CompilerOptions& compiler_options) { - std::unique_ptr<const ArmInstructionSetFeatures> features_arm( - ArmInstructionSetFeatures::FromCppDefines()); - return new (graph->GetArena()) TestCodeGeneratorARM(graph, - *features_arm.get(), - compiler_options); -} - CodeGenerator* create_codegen_arm_vixl32(HGraph* graph, const CompilerOptions& compiler_options) { std::unique_ptr<const ArmInstructionSetFeatures> features_arm( ArmInstructionSetFeatures::FromCppDefines()); diff --git a/compiler/optimizing/common_arm64.h b/compiler/optimizing/common_arm64.h index 721f74eeee..e73fd7ddc8 100644 --- a/compiler/optimizing/common_arm64.h +++ b/compiler/optimizing/common_arm64.h @@ -234,9 +234,20 @@ inline vixl::aarch64::Operand OperandFromMemOperand( } } -inline bool CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* instr) { - DCHECK(constant->IsIntConstant() || constant->IsLongConstant() || constant->IsNullConstant()) - << constant->DebugName(); +inline bool Arm64CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* instr) { + int64_t value = CodeGenerator::GetInt64ValueOf(constant); + + // TODO: Improve this when IsSIMDConstantEncodable method is implemented in VIXL. + if (instr->IsVecReplicateScalar()) { + if (constant->IsLongConstant()) { + return false; + } else if (constant->IsFloatConstant()) { + return vixl::aarch64::Assembler::IsImmFP32(constant->AsFloatConstant()->GetValue()); + } else if (constant->IsDoubleConstant()) { + return vixl::aarch64::Assembler::IsImmFP64(constant->AsDoubleConstant()->GetValue()); + } + return IsUint<8>(value); + } // For single uses we let VIXL handle the constant generation since it will // use registers that are not managed by the register allocator (wip0, wip1). @@ -249,8 +260,6 @@ inline bool CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* inst return true; } - int64_t value = CodeGenerator::GetInt64ValueOf(constant); - if (instr->IsAnd() || instr->IsOr() || instr->IsXor()) { // Uses logical operations. return vixl::aarch64::Assembler::IsImmLogical(value, vixl::aarch64::kXRegSize); @@ -276,7 +285,7 @@ inline bool CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* inst inline Location ARM64EncodableConstantOrRegister(HInstruction* constant, HInstruction* instr) { if (constant->IsConstant() - && CanEncodeConstantAsImmediate(constant->AsConstant(), instr)) { + && Arm64CanEncodeConstantAsImmediate(constant->AsConstant(), instr)) { return Location::ConstantLocation(constant->AsConstant()); } diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.cc b/compiler/optimizing/dex_cache_array_fixups_arm.cc deleted file mode 100644 index 0c832a5c35..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_arm.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "dex_cache_array_fixups_arm.h" - -#include "base/arena_containers.h" -#ifdef ART_USE_OLD_ARM_BACKEND -#include "code_generator_arm.h" -#include "intrinsics_arm.h" -#else -#include "code_generator_arm_vixl.h" -#include "intrinsics_arm_vixl.h" -#endif -#include "utils/dex_cache_arrays_layout-inl.h" - -namespace art { -namespace arm { -#ifdef ART_USE_OLD_ARM_BACKEND -typedef CodeGeneratorARM CodeGeneratorARMType; -typedef IntrinsicLocationsBuilderARM IntrinsicLocationsBuilderARMType; -#else -typedef CodeGeneratorARMVIXL CodeGeneratorARMType; -typedef IntrinsicLocationsBuilderARMVIXL IntrinsicLocationsBuilderARMType; -#endif - -/** - * Finds instructions that need the dex cache arrays base as an input. - */ -class DexCacheArrayFixupsVisitor : public HGraphVisitor { - public: - DexCacheArrayFixupsVisitor(HGraph* graph, CodeGenerator* codegen) - : HGraphVisitor(graph), - codegen_(down_cast<CodeGeneratorARMType*>(codegen)), - dex_cache_array_bases_(std::less<const DexFile*>(), - // Attribute memory use to code generator. - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {} - - void MoveBasesIfNeeded() { - for (const auto& entry : dex_cache_array_bases_) { - // Bring the base closer to the first use (previously, it was in the - // entry block) and relieve some pressure on the register allocator - // while avoiding recalculation of the base in a loop. - HArmDexCacheArraysBase* base = entry.second; - base->MoveBeforeFirstUserAndOutOfLoops(); - } - } - - private: - void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { - // If this is an invoke with PC-relative access to the dex cache methods array, - // we need to add the dex cache arrays base as the special input. - if (invoke->HasPcRelativeDexCache() && - !IsCallFreeIntrinsic<IntrinsicLocationsBuilderARMType>(invoke, codegen_)) { - HArmDexCacheArraysBase* base = - GetOrCreateDexCacheArrayBase(invoke, invoke->GetDexFileForPcRelativeDexCache()); - // Update the element offset in base. - DexCacheArraysLayout layout(kArmPointerSize, &invoke->GetDexFileForPcRelativeDexCache()); - base->UpdateElementOffset(layout.MethodOffset(invoke->GetDexMethodIndex())); - // Add the special argument base to the method. - DCHECK(!invoke->HasCurrentMethodInput()); - invoke->AddSpecialInput(base); - } - } - - HArmDexCacheArraysBase* GetOrCreateDexCacheArrayBase(HInstruction* cursor, - const DexFile& dex_file) { - if (GetGraph()->HasIrreducibleLoops()) { - HArmDexCacheArraysBase* base = new (GetGraph()->GetArena()) HArmDexCacheArraysBase(dex_file); - cursor->GetBlock()->InsertInstructionBefore(base, cursor); - return base; - } else { - // Ensure we only initialize the pointer once for each dex file. - auto lb = dex_cache_array_bases_.lower_bound(&dex_file); - if (lb != dex_cache_array_bases_.end() && - !dex_cache_array_bases_.key_comp()(&dex_file, lb->first)) { - return lb->second; - } - - // Insert the base at the start of the entry block, move it to a better - // position later in MoveBaseIfNeeded(). - HArmDexCacheArraysBase* base = new (GetGraph()->GetArena()) HArmDexCacheArraysBase(dex_file); - HBasicBlock* entry_block = GetGraph()->GetEntryBlock(); - entry_block->InsertInstructionBefore(base, entry_block->GetFirstInstruction()); - dex_cache_array_bases_.PutBefore(lb, &dex_file, base); - return base; - } - } - - CodeGeneratorARMType* codegen_; - - using DexCacheArraysBaseMap = - ArenaSafeMap<const DexFile*, HArmDexCacheArraysBase*, std::less<const DexFile*>>; - DexCacheArraysBaseMap dex_cache_array_bases_; -}; - -void DexCacheArrayFixups::Run() { - DexCacheArrayFixupsVisitor visitor(graph_, codegen_); - visitor.VisitInsertionOrder(); - visitor.MoveBasesIfNeeded(); -} - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.h b/compiler/optimizing/dex_cache_array_fixups_arm.h deleted file mode 100644 index 9d67a319b9..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_arm.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_ARM_H_ -#define ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_ARM_H_ - -#include "nodes.h" -#include "optimization.h" - -namespace art { - -class CodeGenerator; - -namespace arm { - -class DexCacheArrayFixups : public HOptimization { - public: - DexCacheArrayFixups(HGraph* graph, CodeGenerator* codegen, OptimizingCompilerStats* stats) - : HOptimization(graph, kDexCacheArrayFixupsArmPassName, stats), - codegen_(codegen) {} - - static constexpr const char* kDexCacheArrayFixupsArmPassName = "dex_cache_array_fixups_arm"; - - void Run() OVERRIDE; - - private: - CodeGenerator* codegen_; -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_ARM_H_ diff --git a/compiler/optimizing/dex_cache_array_fixups_mips.cc b/compiler/optimizing/dex_cache_array_fixups_mips.cc deleted file mode 100644 index 7734f9197d..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_mips.cc +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "code_generator_mips.h" -#include "dex_cache_array_fixups_mips.h" - -#include "base/arena_containers.h" -#include "intrinsics_mips.h" -#include "utils/dex_cache_arrays_layout-inl.h" - -namespace art { -namespace mips { - -/** - * Finds instructions that need the dex cache arrays base as an input. - */ -class DexCacheArrayFixupsVisitor : public HGraphVisitor { - public: - explicit DexCacheArrayFixupsVisitor(HGraph* graph, CodeGenerator* codegen) - : HGraphVisitor(graph), - codegen_(down_cast<CodeGeneratorMIPS*>(codegen)), - dex_cache_array_bases_(std::less<const DexFile*>(), - // Attribute memory use to code generator. - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {} - - void MoveBasesIfNeeded() { - for (const auto& entry : dex_cache_array_bases_) { - // Bring the base closer to the first use (previously, it was in the - // entry block) and relieve some pressure on the register allocator - // while avoiding recalculation of the base in a loop. - HMipsDexCacheArraysBase* base = entry.second; - base->MoveBeforeFirstUserAndOutOfLoops(); - } - // Computing the dex cache base for PC-relative accesses will clobber RA with - // the NAL instruction on R2. Take a note of this before generating the method - // entry. - if (!dex_cache_array_bases_.empty()) { - codegen_->ClobberRA(); - } - } - - private: - void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { - // If this is an invoke with PC-relative access to the dex cache methods array, - // we need to add the dex cache arrays base as the special input. - if (invoke->HasPcRelativeDexCache() && - !IsCallFreeIntrinsic<IntrinsicLocationsBuilderMIPS>(invoke, codegen_)) { - // Initialize base for target method dex file if needed. - HMipsDexCacheArraysBase* base = - GetOrCreateDexCacheArrayBase(invoke->GetDexFileForPcRelativeDexCache()); - // Update the element offset in base. - DexCacheArraysLayout layout(kMipsPointerSize, &invoke->GetDexFileForPcRelativeDexCache()); - base->UpdateElementOffset(layout.MethodOffset(invoke->GetDexMethodIndex())); - // Add the special argument base to the method. - DCHECK(!invoke->HasCurrentMethodInput()); - invoke->AddSpecialInput(base); - } - } - - HMipsDexCacheArraysBase* GetOrCreateDexCacheArrayBase(const DexFile& dex_file) { - return dex_cache_array_bases_.GetOrCreate( - &dex_file, - [this, &dex_file]() { - HMipsDexCacheArraysBase* base = - new (GetGraph()->GetArena()) HMipsDexCacheArraysBase(dex_file); - HBasicBlock* entry_block = GetGraph()->GetEntryBlock(); - // Insert the base at the start of the entry block, move it to a better - // position later in MoveBaseIfNeeded(). - entry_block->InsertInstructionBefore(base, entry_block->GetFirstInstruction()); - return base; - }); - } - - CodeGeneratorMIPS* codegen_; - - using DexCacheArraysBaseMap = - ArenaSafeMap<const DexFile*, HMipsDexCacheArraysBase*, std::less<const DexFile*>>; - DexCacheArraysBaseMap dex_cache_array_bases_; -}; - -void DexCacheArrayFixups::Run() { - CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen_); - if (mips_codegen->GetInstructionSetFeatures().IsR6()) { - // Do nothing for R6 because it has PC-relative addressing. - return; - } - if (graph_->HasIrreducibleLoops()) { - // Do not run this optimization, as irreducible loops do not work with an instruction - // that can be live-in at the irreducible loop header. - return; - } - DexCacheArrayFixupsVisitor visitor(graph_, codegen_); - visitor.VisitInsertionOrder(); - visitor.MoveBasesIfNeeded(); -} - -} // namespace mips -} // namespace art diff --git a/compiler/optimizing/dex_cache_array_fixups_mips.h b/compiler/optimizing/dex_cache_array_fixups_mips.h deleted file mode 100644 index 861a199d6c..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_mips.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_MIPS_H_ -#define ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_MIPS_H_ - -#include "nodes.h" -#include "optimization.h" - -namespace art { - -class CodeGenerator; - -namespace mips { - -class DexCacheArrayFixups : public HOptimization { - public: - DexCacheArrayFixups(HGraph* graph, CodeGenerator* codegen, OptimizingCompilerStats* stats) - : HOptimization(graph, kDexCacheArrayFixupsMipsPassName, stats), - codegen_(codegen) {} - - static constexpr const char* kDexCacheArrayFixupsMipsPassName = "dex_cache_array_fixups_mips"; - - void Run() OVERRIDE; - - private: - CodeGenerator* codegen_; -}; - -} // namespace mips -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_MIPS_H_ diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc index 02816cf7ce..a20ec3c0db 100644 --- a/compiler/optimizing/graph_visualizer.cc +++ b/compiler/optimizing/graph_visualizer.cc @@ -34,6 +34,7 @@ #include "register_allocator_linear_scan.h" #include "ssa_liveness_analysis.h" #include "utils/assembler.h" +#include "utils/intrusive_forward_list.h" namespace art { @@ -66,6 +67,13 @@ class StringList { current->Dump(NewEntryStream()); } } + // Construct StringList from a list of elements. The value type must provide method `Dump`. + template <typename Container> + explicit StringList(const Container& list, Format format = kArrayBrackets) : StringList(format) { + for (const typename Container::value_type& current : list) { + current.Dump(NewEntryStream()); + } + } std::ostream& NewEntryStream() { if (is_empty_) { @@ -443,8 +451,16 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { void VisitInvoke(HInvoke* invoke) OVERRIDE { StartAttributeStream("dex_file_index") << invoke->GetDexMethodIndex(); - StartAttributeStream("method_name") << GetGraph()->GetDexFile().PrettyMethod( - invoke->GetDexMethodIndex(), /* with_signature */ false); + ArtMethod* method = invoke->GetResolvedMethod(); + // We don't print signatures, which conflict with c1visualizer format. + static constexpr bool kWithSignature = false; + // Note that we can only use the graph's dex file for the unresolved case. The + // other invokes might be coming from inlined methods. + ScopedObjectAccess soa(Thread::Current()); + std::string method_name = (method == nullptr) + ? GetGraph()->GetDexFile().PrettyMethod(invoke->GetDexMethodIndex(), kWithSignature) + : method->PrettyMethod(kWithSignature); + StartAttributeStream("method_name") << method_name; } void VisitInvokeUnresolved(HInvokeUnresolved* invoke) OVERRIDE { @@ -584,8 +600,8 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { LiveInterval* interval = instruction->GetLiveInterval(); StartAttributeStream("ranges") << StringList(interval->GetFirstRange(), StringList::kSetBrackets); - StartAttributeStream("uses") << StringList(interval->GetFirstUse()); - StartAttributeStream("env_uses") << StringList(interval->GetFirstEnvironmentUse()); + StartAttributeStream("uses") << StringList(interval->GetUses()); + StartAttributeStream("env_uses") << StringList(interval->GetEnvironmentUses()); StartAttributeStream("is_fixed") << interval->IsFixed(); StartAttributeStream("is_split") << interval->IsSplit(); StartAttributeStream("is_low") << interval->IsLowInterval(); diff --git a/compiler/optimizing/induction_var_analysis.cc b/compiler/optimizing/induction_var_analysis.cc index 88473f02e5..84b20f65e3 100644 --- a/compiler/optimizing/induction_var_analysis.cc +++ b/compiler/optimizing/induction_var_analysis.cc @@ -695,8 +695,8 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveOp(HLoopInform /*fetch*/ nullptr, type_); default: - CHECK(false) << op; - break; + LOG(FATAL) << op; + UNREACHABLE(); } } } diff --git a/compiler/optimizing/induction_var_range.cc b/compiler/optimizing/induction_var_range.cc index c0ec58f824..f35aace3a9 100644 --- a/compiler/optimizing/induction_var_range.cc +++ b/compiler/optimizing/induction_var_range.cc @@ -373,21 +373,23 @@ bool InductionVarRange::IsFinite(HLoopInformation* loop, /*out*/ int64_t* tc) co bool InductionVarRange::IsUnitStride(HInstruction* context, HInstruction* instruction, + HGraph* graph, /*out*/ HInstruction** offset) const { HLoopInformation* loop = nullptr; HInductionVarAnalysis::InductionInfo* info = nullptr; HInductionVarAnalysis::InductionInfo* trip = nullptr; if (HasInductionInfo(context, instruction, &loop, &info, &trip)) { if (info->induction_class == HInductionVarAnalysis::kLinear && - info->op_b->operation == HInductionVarAnalysis::kFetch && !HInductionVarAnalysis::IsNarrowingLinear(info)) { int64_t stride_value = 0; if (IsConstant(info->op_a, kExact, &stride_value) && stride_value == 1) { int64_t off_value = 0; - if (IsConstant(info->op_b, kExact, &off_value) && off_value == 0) { - *offset = nullptr; - } else { + if (IsConstant(info->op_b, kExact, &off_value)) { + *offset = graph->GetConstant(info->op_b->type, off_value); + } else if (info->op_b->operation == HInductionVarAnalysis::kFetch) { *offset = info->op_b->fetch; + } else { + return false; } return true; } diff --git a/compiler/optimizing/induction_var_range.h b/compiler/optimizing/induction_var_range.h index a8ee829d08..ab1772bf15 100644 --- a/compiler/optimizing/induction_var_range.h +++ b/compiler/optimizing/induction_var_range.h @@ -163,6 +163,7 @@ class InductionVarRange { */ bool IsUnitStride(HInstruction* context, HInstruction* instruction, + HGraph* graph, /*out*/ HInstruction** offset) const; /** diff --git a/compiler/optimizing/induction_var_range_test.cc b/compiler/optimizing/induction_var_range_test.cc index d01d3146fc..67d2093829 100644 --- a/compiler/optimizing/induction_var_range_test.cc +++ b/compiler/optimizing/induction_var_range_test.cc @@ -770,8 +770,8 @@ TEST_F(InductionVarRangeTest, ConstantTripCountUp) { EXPECT_TRUE(range_.IsFinite(loop_header_->GetLoopInformation(), &tc)); EXPECT_EQ(1000, tc); HInstruction* offset = nullptr; - EXPECT_TRUE(range_.IsUnitStride(phi, phi, &offset)); - EXPECT_TRUE(offset == nullptr); + EXPECT_TRUE(range_.IsUnitStride(phi, phi, graph_, &offset)); + ExpectInt(0, offset); HInstruction* tce = range_.GenerateTripCount( loop_header_->GetLoopInformation(), graph_, loop_preheader_); ASSERT_TRUE(tce != nullptr); @@ -826,7 +826,7 @@ TEST_F(InductionVarRangeTest, ConstantTripCountDown) { EXPECT_TRUE(range_.IsFinite(loop_header_->GetLoopInformation(), &tc)); EXPECT_EQ(1000, tc); HInstruction* offset = nullptr; - EXPECT_FALSE(range_.IsUnitStride(phi, phi, &offset)); + EXPECT_FALSE(range_.IsUnitStride(phi, phi, graph_, &offset)); HInstruction* tce = range_.GenerateTripCount( loop_header_->GetLoopInformation(), graph_, loop_preheader_); ASSERT_TRUE(tce != nullptr); @@ -908,8 +908,8 @@ TEST_F(InductionVarRangeTest, SymbolicTripCountUp) { EXPECT_TRUE(range_.IsFinite(loop_header_->GetLoopInformation(), &tc)); EXPECT_EQ(0, tc); // unknown HInstruction* offset = nullptr; - EXPECT_TRUE(range_.IsUnitStride(phi, phi, &offset)); - EXPECT_TRUE(offset == nullptr); + EXPECT_TRUE(range_.IsUnitStride(phi, phi, graph_, &offset)); + ExpectInt(0, offset); HInstruction* tce = range_.GenerateTripCount( loop_header_->GetLoopInformation(), graph_, loop_preheader_); ASSERT_TRUE(tce != nullptr); @@ -994,7 +994,7 @@ TEST_F(InductionVarRangeTest, SymbolicTripCountDown) { EXPECT_TRUE(range_.IsFinite(loop_header_->GetLoopInformation(), &tc)); EXPECT_EQ(0, tc); // unknown HInstruction* offset = nullptr; - EXPECT_FALSE(range_.IsUnitStride(phi, phi, &offset)); + EXPECT_FALSE(range_.IsUnitStride(phi, phi, graph_, &offset)); HInstruction* tce = range_.GenerateTripCount( loop_header_->GetLoopInformation(), graph_, loop_preheader_); ASSERT_TRUE(tce != nullptr); diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 142c95780e..18390cc4d4 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -146,7 +146,10 @@ void HInliner::Run() { // that this method is actually inlined; // - if a method's name contains the substring "$noinline$", do not // inline that method. - const bool honor_inlining_directives = IsCompilingWithCoreImage(); + // We limit this to AOT compilation, as the JIT may or may not inline + // depending on the state of classes at runtime. + const bool honor_inlining_directives = + IsCompilingWithCoreImage() && Runtime::Current()->IsAotCompiler(); // Keep a copy of all blocks when starting the visit. ArenaVector<HBasicBlock*> blocks = graph_->GetReversePostOrder(); diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index df9e7164ed..8054140924 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -22,6 +22,7 @@ #include "dex_instruction-inl.h" #include "driver/compiler_options.h" #include "imtable-inl.h" +#include "quicken_info.h" #include "sharpening.h" #include "scoped_thread_state_change-inl.h" @@ -312,6 +313,11 @@ bool HInstructionBuilder::Build() { DCHECK(!IsBlockPopulated(current_block_)); + uint32_t quicken_index = 0; + if (CanDecodeQuickenedInfo()) { + quicken_index = block_builder_->GetQuickenIndex(block_dex_pc); + } + for (CodeItemIterator it(code_item_, block_dex_pc); !it.Done(); it.Advance()) { if (current_block_ == nullptr) { // The previous instruction ended this block. @@ -332,9 +338,13 @@ bool HInstructionBuilder::Build() { AppendInstruction(new (arena_) HNativeDebugInfo(dex_pc)); } - if (!ProcessDexInstruction(it.CurrentInstruction(), dex_pc)) { + if (!ProcessDexInstruction(it.CurrentInstruction(), dex_pc, quicken_index)) { return false; } + + if (QuickenInfoTable::NeedsIndexForInstruction(&it.CurrentInstruction())) { + ++quicken_index; + } } if (current_block_ != nullptr) { @@ -654,10 +664,7 @@ void HInstructionBuilder::BuildReturn(const Instruction& instruction, // TODO: remove redundant constructor fences (b/36656456). if (RequiresConstructorBarrier(dex_compilation_unit_, compiler_driver_)) { // Compiling instance constructor. - if (kIsDebugBuild) { - std::string method_name = graph_->GetMethodName(); - CHECK_EQ(std::string("<init>"), method_name); - } + DCHECK_STREQ("<init>", graph_->GetMethodName()); HInstruction* fence_target = current_this_parameter_; DCHECK(fence_target != nullptr); @@ -700,29 +707,18 @@ static InvokeType GetInvokeTypeFromOpCode(Instruction::Code opcode) { ArtMethod* HInstructionBuilder::ResolveMethod(uint16_t method_idx, InvokeType invoke_type) { ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<2> hs(soa.Self()); ClassLinker* class_linker = dex_compilation_unit_->GetClassLinker(); Handle<mirror::ClassLoader> class_loader = dex_compilation_unit_->GetClassLoader(); - Handle<mirror::Class> compiling_class(hs.NewHandle(GetCompilingClass())); - // We fetch the referenced class eagerly (that is, the class pointed by in the MethodId - // at method_idx), as `CanAccessResolvedMethod` expects it be be in the dex cache. - Handle<mirror::Class> methods_class(hs.NewHandle(class_linker->ResolveReferencedClassOfMethod( - method_idx, dex_compilation_unit_->GetDexCache(), class_loader))); - if (UNLIKELY(methods_class == nullptr)) { - // Clean up any exception left by type resolution. - soa.Self()->ClearException(); - return nullptr; - } - - ArtMethod* resolved_method = class_linker->ResolveMethod<ClassLinker::kForceICCECheck>( - *dex_compilation_unit_->GetDexFile(), - method_idx, - dex_compilation_unit_->GetDexCache(), - class_loader, - /* referrer */ nullptr, - invoke_type); + ArtMethod* resolved_method = + class_linker->ResolveMethod<ClassLinker::ResolveMode::kCheckICCEAndIAE>( + *dex_compilation_unit_->GetDexFile(), + method_idx, + dex_compilation_unit_->GetDexCache(), + class_loader, + graph_->GetArtMethod(), + invoke_type); if (UNLIKELY(resolved_method == nullptr)) { // Clean up any exception left by type resolution. @@ -730,17 +726,14 @@ ArtMethod* HInstructionBuilder::ResolveMethod(uint16_t method_idx, InvokeType in return nullptr; } - // Check access. The class linker has a fast path for looking into the dex cache - // and does not check the access if it hits it. - if (compiling_class == nullptr) { + // The referrer may be unresolved for AOT if we're compiling a class that cannot be + // resolved because, for example, we don't find a superclass in the classpath. + if (graph_->GetArtMethod() == nullptr) { + // The class linker cannot check access without a referrer, so we have to do it. + // Fall back to HInvokeUnresolved if the method isn't public. if (!resolved_method->IsPublic()) { return nullptr; } - } else if (!compiling_class->CanAccessResolvedMethod(resolved_method->GetDeclaringClass(), - resolved_method, - dex_compilation_unit_->GetDexCache().Get(), - method_idx)) { - return nullptr; } // We have to special case the invoke-super case, as ClassLinker::ResolveMethod does not. @@ -748,19 +741,26 @@ ArtMethod* HInstructionBuilder::ResolveMethod(uint16_t method_idx, InvokeType in // make this an invoke-unresolved to handle cross-dex invokes or abstract super methods, both of // which require runtime handling. if (invoke_type == kSuper) { + ObjPtr<mirror::Class> compiling_class = GetCompilingClass(); if (compiling_class == nullptr) { // We could not determine the method's class we need to wait until runtime. DCHECK(Runtime::Current()->IsAotCompiler()); return nullptr; } - if (!methods_class->IsAssignableFrom(compiling_class.Get())) { + ObjPtr<mirror::Class> referenced_class = class_linker->LookupResolvedType( + *dex_compilation_unit_->GetDexFile(), + dex_compilation_unit_->GetDexFile()->GetMethodId(method_idx).class_idx_, + dex_compilation_unit_->GetDexCache().Get(), + class_loader.Get()); + DCHECK(referenced_class != nullptr); // We have already resolved a method from this class. + if (!referenced_class->IsAssignableFrom(compiling_class)) { // We cannot statically determine the target method. The runtime will throw a // NoSuchMethodError on this one. return nullptr; } ArtMethod* actual_method; - if (methods_class->IsInterface()) { - actual_method = methods_class->FindVirtualMethodForInterfaceSuper( + if (referenced_class->IsInterface()) { + actual_method = referenced_class->FindVirtualMethodForInterfaceSuper( resolved_method, class_linker->GetImagePointerSize()); } else { uint16_t vtable_index = resolved_method->GetMethodIndex(); @@ -787,12 +787,6 @@ ArtMethod* HInstructionBuilder::ResolveMethod(uint16_t method_idx, InvokeType in resolved_method = actual_method; } - // Check for incompatible class changes. The class linker has a fast path for - // looking into the dex cache and does not check incompatible class changes if it hits it. - if (resolved_method->CheckIncompatibleClassChange(invoke_type)) { - return nullptr; - } - return resolved_method; } @@ -888,7 +882,7 @@ bool HInstructionBuilder::BuildInvoke(const Instruction& instruction, } HInvokeStaticOrDirect::DispatchInfo dispatch_info = { - HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod, + HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall, HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, 0u }; @@ -1261,7 +1255,8 @@ static Primitive::Type GetFieldAccessType(const DexFile& dex_file, uint16_t fiel bool HInstructionBuilder::BuildInstanceFieldAccess(const Instruction& instruction, uint32_t dex_pc, - bool is_put) { + bool is_put, + size_t quicken_index) { uint32_t source_or_dest_reg = instruction.VRegA_22c(); uint32_t obj_reg = instruction.VRegB_22c(); uint16_t field_index; @@ -1269,7 +1264,7 @@ bool HInstructionBuilder::BuildInstanceFieldAccess(const Instruction& instructio if (!CanDecodeQuickenedInfo()) { return false; } - field_index = LookupQuickenedInfo(dex_pc); + field_index = LookupQuickenedInfo(quicken_index); } else { field_index = instruction.VRegC_22c(); } @@ -1805,40 +1800,17 @@ bool HInstructionBuilder::NeedsAccessCheck(dex::TypeIndex type_index, bool* fina } bool HInstructionBuilder::CanDecodeQuickenedInfo() const { - return interpreter_metadata_ != nullptr; + return !quicken_info_.IsNull(); } -uint16_t HInstructionBuilder::LookupQuickenedInfo(uint32_t dex_pc) { - DCHECK(interpreter_metadata_ != nullptr); - - // First check if the info has already been decoded from `interpreter_metadata_`. - auto it = skipped_interpreter_metadata_.find(dex_pc); - if (it != skipped_interpreter_metadata_.end()) { - // Remove the entry from the map and return the parsed info. - uint16_t value_in_map = it->second; - skipped_interpreter_metadata_.erase(it); - return value_in_map; - } - - // Otherwise start parsing `interpreter_metadata_` until the slot for `dex_pc` - // is found. Store skipped values in the `skipped_interpreter_metadata_` map. - while (true) { - uint32_t dex_pc_in_map = DecodeUnsignedLeb128(&interpreter_metadata_); - uint16_t value_in_map = DecodeUnsignedLeb128(&interpreter_metadata_); - DCHECK_LE(dex_pc_in_map, dex_pc); - - if (dex_pc_in_map == dex_pc) { - return value_in_map; - } else { - // Overwrite and not Put, as quickened CHECK-CAST has two entries with - // the same dex_pc. This is OK, because the compiler does not care about those - // entries. - skipped_interpreter_metadata_.Overwrite(dex_pc_in_map, value_in_map); - } - } +uint16_t HInstructionBuilder::LookupQuickenedInfo(uint32_t quicken_index) { + DCHECK(CanDecodeQuickenedInfo()); + return quicken_info_.GetData(quicken_index); } -bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, uint32_t dex_pc) { +bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, + uint32_t dex_pc, + size_t quicken_index) { switch (instruction.Opcode()) { case Instruction::CONST_4: { int32_t register_index = instruction.VRegA(); @@ -1995,7 +1967,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, if (!CanDecodeQuickenedInfo()) { return false; } - method_idx = LookupQuickenedInfo(dex_pc); + method_idx = LookupQuickenedInfo(quicken_index); } else { method_idx = instruction.VRegB_35c(); } @@ -2020,7 +1992,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, if (!CanDecodeQuickenedInfo()) { return false; } - method_idx = LookupQuickenedInfo(dex_pc); + method_idx = LookupQuickenedInfo(quicken_index); } else { method_idx = instruction.VRegB_3rc(); } @@ -2693,7 +2665,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::IGET_CHAR_QUICK: case Instruction::IGET_SHORT: case Instruction::IGET_SHORT_QUICK: { - if (!BuildInstanceFieldAccess(instruction, dex_pc, false)) { + if (!BuildInstanceFieldAccess(instruction, dex_pc, false, quicken_index)) { return false; } break; @@ -2713,7 +2685,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::IPUT_CHAR_QUICK: case Instruction::IPUT_SHORT: case Instruction::IPUT_SHORT_QUICK: { - if (!BuildInstanceFieldAccess(instruction, dex_pc, true)) { + if (!BuildInstanceFieldAccess(instruction, dex_pc, true, quicken_index)) { return false; } break; diff --git a/compiler/optimizing/instruction_builder.h b/compiler/optimizing/instruction_builder.h index e968760d84..5a83df3813 100644 --- a/compiler/optimizing/instruction_builder.h +++ b/compiler/optimizing/instruction_builder.h @@ -27,6 +27,7 @@ #include "mirror/dex_cache.h" #include "nodes.h" #include "optimizing_compiler_stats.h" +#include "quicken_info.h" #include "ssa_builder.h" namespace art { @@ -67,9 +68,7 @@ class HInstructionBuilder : public ValueObject { code_generator_(code_generator), dex_compilation_unit_(dex_compilation_unit), outer_compilation_unit_(outer_compilation_unit), - interpreter_metadata_(interpreter_metadata), - skipped_interpreter_metadata_(std::less<uint32_t>(), - arena_->Adapter(kArenaAllocGraphBuilder)), + quicken_info_(interpreter_metadata), compilation_stats_(compiler_stats), dex_cache_(dex_cache), loop_headers_(graph->GetArena()->Adapter(kArenaAllocGraphBuilder)) { @@ -85,11 +84,11 @@ class HInstructionBuilder : public ValueObject { void PropagateLocalsToCatchBlocks(); void SetLoopHeaderPhiInputs(); - bool ProcessDexInstruction(const Instruction& instruction, uint32_t dex_pc); + bool ProcessDexInstruction(const Instruction& instruction, uint32_t dex_pc, size_t quicken_index); void FindNativeDebugInfoLocations(ArenaBitVector* locations); bool CanDecodeQuickenedInfo() const; - uint16_t LookupQuickenedInfo(uint32_t dex_pc); + uint16_t LookupQuickenedInfo(uint32_t quicken_index); HBasicBlock* FindBlockStartingAt(uint32_t dex_pc) const; @@ -159,7 +158,10 @@ class HInstructionBuilder : public ValueObject { void BuildReturn(const Instruction& instruction, Primitive::Type type, uint32_t dex_pc); // Builds an instance field access node and returns whether the instruction is supported. - bool BuildInstanceFieldAccess(const Instruction& instruction, uint32_t dex_pc, bool is_put); + bool BuildInstanceFieldAccess(const Instruction& instruction, + uint32_t dex_pc, + bool is_put, + size_t quicken_index); void BuildUnresolvedStaticFieldAccess(const Instruction& instruction, uint32_t dex_pc, @@ -349,14 +351,8 @@ class HInstructionBuilder : public ValueObject { // methods. const DexCompilationUnit* const outer_compilation_unit_; - // Original values kept after instruction quickening. This is a data buffer - // of Leb128-encoded (dex_pc, value) pairs sorted by dex_pc. - const uint8_t* interpreter_metadata_; - - // InstructionBuilder does not parse instructions in dex_pc order. Quickening - // info for out-of-order dex_pcs is stored in a map until the positions - // are eventually visited. - ArenaSafeMap<uint32_t, uint16_t> skipped_interpreter_metadata_; + // Original values kept after instruction quickening. + QuickenInfoTable quicken_info_; OptimizingCompilerStats* compilation_stats_; Handle<mirror::DexCache> dex_cache_; diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index d14716601c..f2a8cc0333 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -1867,33 +1867,35 @@ void InstructionSimplifierVisitor::SimplifySystemArrayCopy(HInvoke* instruction) ArtMethod* method = nullptr; switch (source_component_type) { case Primitive::kPrimBoolean: - method = system->FindDeclaredDirectMethod("arraycopy", "([ZI[ZII)V", image_size); + method = system->FindClassMethod("arraycopy", "([ZI[ZII)V", image_size); break; case Primitive::kPrimByte: - method = system->FindDeclaredDirectMethod("arraycopy", "([BI[BII)V", image_size); + method = system->FindClassMethod("arraycopy", "([BI[BII)V", image_size); break; case Primitive::kPrimChar: - method = system->FindDeclaredDirectMethod("arraycopy", "([CI[CII)V", image_size); + method = system->FindClassMethod("arraycopy", "([CI[CII)V", image_size); break; case Primitive::kPrimShort: - method = system->FindDeclaredDirectMethod("arraycopy", "([SI[SII)V", image_size); + method = system->FindClassMethod("arraycopy", "([SI[SII)V", image_size); break; case Primitive::kPrimInt: - method = system->FindDeclaredDirectMethod("arraycopy", "([II[III)V", image_size); + method = system->FindClassMethod("arraycopy", "([II[III)V", image_size); break; case Primitive::kPrimFloat: - method = system->FindDeclaredDirectMethod("arraycopy", "([FI[FII)V", image_size); + method = system->FindClassMethod("arraycopy", "([FI[FII)V", image_size); break; case Primitive::kPrimLong: - method = system->FindDeclaredDirectMethod("arraycopy", "([JI[JII)V", image_size); + method = system->FindClassMethod("arraycopy", "([JI[JII)V", image_size); break; case Primitive::kPrimDouble: - method = system->FindDeclaredDirectMethod("arraycopy", "([DI[DII)V", image_size); + method = system->FindClassMethod("arraycopy", "([DI[DII)V", image_size); break; default: LOG(FATAL) << "Unreachable"; } DCHECK(method != nullptr); + DCHECK(method->IsStatic()); + DCHECK(method->GetDeclaringClass() == system); invoke->SetResolvedMethod(method); // Sharpen the new invoke. Note that we do not update the dex method index of // the invoke, as we would need to look it up in the current dex file, and it diff --git a/compiler/optimizing/instruction_simplifier_arm.cc b/compiler/optimizing/instruction_simplifier_arm.cc index 3fc7c50bb1..fe22595258 100644 --- a/compiler/optimizing/instruction_simplifier_arm.cc +++ b/compiler/optimizing/instruction_simplifier_arm.cc @@ -147,8 +147,8 @@ void InstructionSimplifierArmVisitor::VisitArrayGet(HArrayGet* instruction) { Primitive::Type type = instruction->GetType(); // TODO: Implement reading (length + compression) for String compression feature from - // negative offset (count_offset - data_offset). Thumb2Assembler does not support T4 - // encoding of "LDR (immediate)" at the moment. + // negative offset (count_offset - data_offset). Thumb2Assembler (now removed) did + // not support T4 encoding of "LDR (immediate)", but ArmVIXLMacroAssembler might. // Don't move array pointer if it is charAt because we need to take the count first. if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { return; diff --git a/compiler/optimizing/instruction_simplifier_shared.cc b/compiler/optimizing/instruction_simplifier_shared.cc index e5a8499ff4..d1bc4dadeb 100644 --- a/compiler/optimizing/instruction_simplifier_shared.cc +++ b/compiler/optimizing/instruction_simplifier_shared.cc @@ -274,8 +274,8 @@ bool TryExtractArrayAccessAddress(HInstruction* access, // `HArm64Load` and `HArm64Store`,`HArmLoad` and `HArmStore`). We defer these changes // because these new instructions would not bring any advantages yet. // Also see the comments in - // `InstructionCodeGeneratorARM::VisitArrayGet()` - // `InstructionCodeGeneratorARM::VisitArraySet()` + // `InstructionCodeGeneratorARMVIXL::VisitArrayGet()` + // `InstructionCodeGeneratorARMVIXL::VisitArraySet()` // `InstructionCodeGeneratorARM64::VisitArrayGet()` // `InstructionCodeGeneratorARM64::VisitArraySet()`. return true; diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc deleted file mode 100644 index ae5f8d1760..0000000000 --- a/compiler/optimizing/intrinsics_arm.cc +++ /dev/null @@ -1,2760 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "intrinsics_arm.h" - -#include "arch/arm/instruction_set_features_arm.h" -#include "art_method.h" -#include "code_generator_arm.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "intrinsics.h" -#include "intrinsics_utils.h" -#include "lock_word.h" -#include "mirror/array-inl.h" -#include "mirror/object_array-inl.h" -#include "mirror/reference.h" -#include "mirror/string.h" -#include "scoped_thread_state_change-inl.h" -#include "thread-current-inl.h" -#include "utils/arm/assembler_arm.h" - -namespace art { - -namespace arm { - -ArmAssembler* IntrinsicCodeGeneratorARM::GetAssembler() { - return codegen_->GetAssembler(); -} - -ArenaAllocator* IntrinsicCodeGeneratorARM::GetAllocator() { - return codegen_->GetGraph()->GetArena(); -} - -using IntrinsicSlowPathARM = IntrinsicSlowPath<InvokeDexCallingConventionVisitorARM>; - -#define __ assembler-> - -// Compute base address for the System.arraycopy intrinsic in `base`. -static void GenSystemArrayCopyBaseAddress(ArmAssembler* assembler, - Primitive::Type type, - const Register& array, - const Location& pos, - const Register& base) { - // This routine is only used by the SystemArrayCopy intrinsic at the - // moment. We can allow Primitive::kPrimNot as `type` to implement - // the SystemArrayCopyChar intrinsic. - DCHECK_EQ(type, Primitive::kPrimNot); - const int32_t element_size = Primitive::ComponentSize(type); - const uint32_t element_size_shift = Primitive::ComponentSizeShift(type); - const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value(); - - if (pos.IsConstant()) { - int32_t constant = pos.GetConstant()->AsIntConstant()->GetValue(); - __ AddConstant(base, array, element_size * constant + data_offset); - } else { - __ add(base, array, ShifterOperand(pos.AsRegister<Register>(), LSL, element_size_shift)); - __ AddConstant(base, data_offset); - } -} - -// Compute end address for the System.arraycopy intrinsic in `end`. -static void GenSystemArrayCopyEndAddress(ArmAssembler* assembler, - Primitive::Type type, - const Location& copy_length, - const Register& base, - const Register& end) { - // This routine is only used by the SystemArrayCopy intrinsic at the - // moment. We can allow Primitive::kPrimNot as `type` to implement - // the SystemArrayCopyChar intrinsic. - DCHECK_EQ(type, Primitive::kPrimNot); - const int32_t element_size = Primitive::ComponentSize(type); - const uint32_t element_size_shift = Primitive::ComponentSizeShift(type); - - if (copy_length.IsConstant()) { - int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue(); - __ AddConstant(end, base, element_size * constant); - } else { - __ add(end, base, ShifterOperand(copy_length.AsRegister<Register>(), LSL, element_size_shift)); - } -} - -#undef __ - -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT - -// Slow path implementing the SystemArrayCopy intrinsic copy loop with read barriers. -class ReadBarrierSystemArrayCopySlowPathARM : public SlowPathCode { - public: - explicit ReadBarrierSystemArrayCopySlowPathARM(HInstruction* instruction) - : SlowPathCode(instruction) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - ArmAssembler* assembler = arm_codegen->GetAssembler(); - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(locations->CanCall()); - DCHECK(instruction_->IsInvokeStaticOrDirect()) - << "Unexpected instruction in read barrier arraycopy slow path: " - << instruction_->DebugName(); - DCHECK(instruction_->GetLocations()->Intrinsified()); - DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy); - - Primitive::Type type = Primitive::kPrimNot; - const int32_t element_size = Primitive::ComponentSize(type); - - Register dest = locations->InAt(2).AsRegister<Register>(); - Location dest_pos = locations->InAt(3); - Register src_curr_addr = locations->GetTemp(0).AsRegister<Register>(); - Register dst_curr_addr = locations->GetTemp(1).AsRegister<Register>(); - Register src_stop_addr = locations->GetTemp(2).AsRegister<Register>(); - Register tmp = locations->GetTemp(3).AsRegister<Register>(); - - __ Bind(GetEntryLabel()); - // Compute the base destination address in `dst_curr_addr`. - GenSystemArrayCopyBaseAddress(assembler, type, dest, dest_pos, dst_curr_addr); - - Label loop; - __ Bind(&loop); - __ ldr(tmp, Address(src_curr_addr, element_size, Address::PostIndex)); - __ MaybeUnpoisonHeapReference(tmp); - // TODO: Inline the mark bit check before calling the runtime? - // tmp = ReadBarrier::Mark(tmp); - // No need to save live registers; it's taken care of by the - // entrypoint. Also, there is no need to update the stack mask, - // as this runtime call will not trigger a garbage collection. - // (See ReadBarrierMarkSlowPathARM::EmitNativeCode for more - // explanations.) - DCHECK_NE(tmp, SP); - DCHECK_NE(tmp, LR); - DCHECK_NE(tmp, PC); - // IP is used internally by the ReadBarrierMarkRegX entry point - // as a temporary (and not preserved). It thus cannot be used by - // any live register in this slow path. - DCHECK_NE(src_curr_addr, IP); - DCHECK_NE(dst_curr_addr, IP); - DCHECK_NE(src_stop_addr, IP); - DCHECK_NE(tmp, IP); - DCHECK(0 <= tmp && tmp < kNumberOfCoreRegisters) << tmp; - // TODO: Load the entrypoint once before the loop, instead of - // loading it at every iteration. - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp); - // This runtime call does not require a stack map. - arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); - __ MaybePoisonHeapReference(tmp); - __ str(tmp, Address(dst_curr_addr, element_size, Address::PostIndex)); - __ cmp(src_curr_addr, ShifterOperand(src_stop_addr)); - __ b(&loop, NE); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierSystemArrayCopySlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(ReadBarrierSystemArrayCopySlowPathARM); -}; - -#undef __ - -IntrinsicLocationsBuilderARM::IntrinsicLocationsBuilderARM(CodeGeneratorARM* codegen) - : arena_(codegen->GetGraph()->GetArena()), - codegen_(codegen), - assembler_(codegen->GetAssembler()), - features_(codegen->GetInstructionSetFeatures()) {} - -bool IntrinsicLocationsBuilderARM::TryDispatch(HInvoke* invoke) { - Dispatch(invoke); - LocationSummary* res = invoke->GetLocations(); - if (res == nullptr) { - return false; - } - return res->Intrinsified(); -} - -#define __ assembler-> - -static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresRegister()); -} - -static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); -} - -static void MoveFPToInt(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { - Location input = locations->InAt(0); - Location output = locations->Out(); - if (is64bit) { - __ vmovrrd(output.AsRegisterPairLow<Register>(), - output.AsRegisterPairHigh<Register>(), - FromLowSToD(input.AsFpuRegisterPairLow<SRegister>())); - } else { - __ vmovrs(output.AsRegister<Register>(), input.AsFpuRegister<SRegister>()); - } -} - -static void MoveIntToFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { - Location input = locations->InAt(0); - Location output = locations->Out(); - if (is64bit) { - __ vmovdrr(FromLowSToD(output.AsFpuRegisterPairLow<SRegister>()), - input.AsRegisterPairLow<Register>(), - input.AsRegisterPairHigh<Register>()); - } else { - __ vmovsr(output.AsFpuRegister<SRegister>(), input.AsRegister<Register>()); - } -} - -void IntrinsicLocationsBuilderARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} -void IntrinsicLocationsBuilderARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) { - CreateIntToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { - MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} -void IntrinsicCodeGeneratorARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) { - MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} -void IntrinsicLocationsBuilderARM::VisitFloatIntBitsToFloat(HInvoke* invoke) { - CreateIntToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) { - MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} -void IntrinsicCodeGeneratorARM::VisitFloatIntBitsToFloat(HInvoke* invoke) { - MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); -} - -static void GenNumberOfLeadingZeros(HInvoke* invoke, - Primitive::Type type, - CodeGeneratorARM* codegen) { - ArmAssembler* assembler = codegen->GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - Location in = locations->InAt(0); - Register out = locations->Out().AsRegister<Register>(); - - DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); - - if (type == Primitive::kPrimLong) { - Register in_reg_lo = in.AsRegisterPairLow<Register>(); - Register in_reg_hi = in.AsRegisterPairHigh<Register>(); - Label end; - Label* final_label = codegen->GetFinalLabel(invoke, &end); - __ clz(out, in_reg_hi); - __ CompareAndBranchIfNonZero(in_reg_hi, final_label); - __ clz(out, in_reg_lo); - __ AddConstant(out, 32); - if (end.IsLinked()) { - __ Bind(&end); - } - } else { - __ clz(out, in.AsRegister<Register>()); - } -} - -void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { - GenNumberOfLeadingZeros(invoke, Primitive::kPrimInt, codegen_); -} - -void IntrinsicLocationsBuilderARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { - GenNumberOfLeadingZeros(invoke, Primitive::kPrimLong, codegen_); -} - -static void GenNumberOfTrailingZeros(HInvoke* invoke, - Primitive::Type type, - CodeGeneratorARM* codegen) { - DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); - - ArmAssembler* assembler = codegen->GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - Register out = locations->Out().AsRegister<Register>(); - - if (type == Primitive::kPrimLong) { - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Label end; - Label* final_label = codegen->GetFinalLabel(invoke, &end); - __ rbit(out, in_reg_lo); - __ clz(out, out); - __ CompareAndBranchIfNonZero(in_reg_lo, final_label); - __ rbit(out, in_reg_hi); - __ clz(out, out); - __ AddConstant(out, 32); - if (end.IsLinked()) { - __ Bind(&end); - } - } else { - Register in = locations->InAt(0).AsRegister<Register>(); - __ rbit(out, in); - __ clz(out, out); - } -} - -void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { - GenNumberOfTrailingZeros(invoke, Primitive::kPrimInt, codegen_); -} - -void IntrinsicLocationsBuilderARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { - GenNumberOfTrailingZeros(invoke, Primitive::kPrimLong, codegen_); -} - -static void MathAbsFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { - Location in = locations->InAt(0); - Location out = locations->Out(); - - if (is64bit) { - __ vabsd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - } else { - __ vabss(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); - } -} - -void IntrinsicLocationsBuilderARM::VisitMathAbsDouble(HInvoke* invoke) { - CreateFPToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitMathAbsFloat(HInvoke* invoke) { - CreateFPToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -static void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - - locations->AddTemp(Location::RequiresRegister()); -} - -static void GenAbsInteger(LocationSummary* locations, - bool is64bit, - ArmAssembler* assembler) { - Location in = locations->InAt(0); - Location output = locations->Out(); - - Register mask = locations->GetTemp(0).AsRegister<Register>(); - - if (is64bit) { - Register in_reg_lo = in.AsRegisterPairLow<Register>(); - Register in_reg_hi = in.AsRegisterPairHigh<Register>(); - Register out_reg_lo = output.AsRegisterPairLow<Register>(); - Register out_reg_hi = output.AsRegisterPairHigh<Register>(); - - DCHECK_NE(out_reg_lo, in_reg_hi) << "Diagonal overlap unexpected."; - - __ Asr(mask, in_reg_hi, 31); - __ adds(out_reg_lo, in_reg_lo, ShifterOperand(mask)); - __ adc(out_reg_hi, in_reg_hi, ShifterOperand(mask)); - __ eor(out_reg_lo, mask, ShifterOperand(out_reg_lo)); - __ eor(out_reg_hi, mask, ShifterOperand(out_reg_hi)); - } else { - Register in_reg = in.AsRegister<Register>(); - Register out_reg = output.AsRegister<Register>(); - - __ Asr(mask, in_reg, 31); - __ add(out_reg, in_reg, ShifterOperand(mask)); - __ eor(out_reg, mask, ShifterOperand(out_reg)); - } -} - -void IntrinsicLocationsBuilderARM::VisitMathAbsInt(HInvoke* invoke) { - CreateIntToIntPlusTemp(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - - -void IntrinsicLocationsBuilderARM::VisitMathAbsLong(HInvoke* invoke) { - CreateIntToIntPlusTemp(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsLong(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -static void GenMinMax(LocationSummary* locations, - bool is_min, - ArmAssembler* assembler) { - Register op1 = locations->InAt(0).AsRegister<Register>(); - Register op2 = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - __ cmp(op1, ShifterOperand(op2)); - - __ it((is_min) ? Condition::LT : Condition::GT, kItElse); - __ mov(out, ShifterOperand(op1), is_min ? Condition::LT : Condition::GT); - __ mov(out, ShifterOperand(op2), is_min ? Condition::GE : Condition::LE); -} - -static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicLocationsBuilderARM::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitMathSqrt(HInvoke* invoke) { - CreateFPToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathSqrt(HInvoke* invoke) { - LocationSummary* locations = invoke->GetLocations(); - ArmAssembler* assembler = GetAssembler(); - __ vsqrtd(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekByte(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekByte(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - __ ldrsb(invoke->GetLocations()->Out().AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekIntNative(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekIntNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - __ ldr(invoke->GetLocations()->Out().AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekLongNative(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekLongNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>(); - // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor - // exception. So we can't use ldrd as addr may be unaligned. - Register lo = invoke->GetLocations()->Out().AsRegisterPairLow<Register>(); - Register hi = invoke->GetLocations()->Out().AsRegisterPairHigh<Register>(); - if (addr == lo) { - __ ldr(hi, Address(addr, 4)); - __ ldr(lo, Address(addr, 0)); - } else { - __ ldr(lo, Address(addr, 0)); - __ ldr(hi, Address(addr, 4)); - } -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekShortNative(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekShortNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - __ ldrsh(invoke->GetLocations()->Out().AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeByte(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeByte(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ strb(invoke->GetLocations()->InAt(1).AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeIntNative(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeIntNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ str(invoke->GetLocations()->InAt(1).AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeLongNative(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeLongNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>(); - // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor - // exception. So we can't use ldrd as addr may be unaligned. - __ str(invoke->GetLocations()->InAt(1).AsRegisterPairLow<Register>(), Address(addr, 0)); - __ str(invoke->GetLocations()->InAt(1).AsRegisterPairHigh<Register>(), Address(addr, 4)); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeShortNative(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeShortNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ strh(invoke->GetLocations()->InAt(1).AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitThreadCurrentThread(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetOut(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitThreadCurrentThread(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ LoadFromOffset(kLoadWord, - invoke->GetLocations()->Out().AsRegister<Register>(), - TR, - Thread::PeerOffset<kArmPointerSize>().Int32Value()); -} - -static void GenUnsafeGet(HInvoke* invoke, - Primitive::Type type, - bool is_volatile, - CodeGeneratorARM* codegen) { - LocationSummary* locations = invoke->GetLocations(); - ArmAssembler* assembler = codegen->GetAssembler(); - Location base_loc = locations->InAt(1); - Register base = base_loc.AsRegister<Register>(); // Object pointer. - Location offset_loc = locations->InAt(2); - Register offset = offset_loc.AsRegisterPairLow<Register>(); // Long offset, lo part only. - Location trg_loc = locations->Out(); - - switch (type) { - case Primitive::kPrimInt: { - Register trg = trg_loc.AsRegister<Register>(); - __ ldr(trg, Address(base, offset)); - if (is_volatile) { - __ dmb(ISH); - } - break; - } - - case Primitive::kPrimNot: { - Register trg = trg_loc.AsRegister<Register>(); - if (kEmitCompilerReadBarrier) { - if (kUseBakerReadBarrier) { - Location temp = locations->GetTemp(0); - codegen->GenerateReferenceLoadWithBakerReadBarrier( - invoke, trg_loc, base, 0U, offset_loc, TIMES_1, temp, /* needs_null_check */ false); - if (is_volatile) { - __ dmb(ISH); - } - } else { - __ ldr(trg, Address(base, offset)); - if (is_volatile) { - __ dmb(ISH); - } - codegen->GenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc); - } - } else { - __ ldr(trg, Address(base, offset)); - if (is_volatile) { - __ dmb(ISH); - } - __ MaybeUnpoisonHeapReference(trg); - } - break; - } - - case Primitive::kPrimLong: { - Register trg_lo = trg_loc.AsRegisterPairLow<Register>(); - __ add(IP, base, ShifterOperand(offset)); - if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { - Register trg_hi = trg_loc.AsRegisterPairHigh<Register>(); - __ ldrexd(trg_lo, trg_hi, IP); - } else { - __ ldrd(trg_lo, Address(IP)); - } - if (is_volatile) { - __ dmb(ISH); - } - break; - } - - default: - LOG(FATAL) << "Unexpected type " << type; - UNREACHABLE(); - } -} - -static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, - HInvoke* invoke, - Primitive::Type type) { - bool can_call = kEmitCompilerReadBarrier && - (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || - invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); - LocationSummary* locations = new (arena) LocationSummary(invoke, - (can_call - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall), - kIntrinsified); - if (can_call && kUseBakerReadBarrier) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::NoLocation()); // Unused receiver. - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), - (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); - if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // We need a temporary register for the read barrier marking slow - // path in InstructionCodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier. - locations->AddTemp(Location::RequiresRegister()); - } -} - -void IntrinsicLocationsBuilderARM::VisitUnsafeGet(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetLong(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetObject(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); -} - -void IntrinsicCodeGeneratorARM::VisitUnsafeGet(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetLong(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetObject(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_); -} - -static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, - const ArmInstructionSetFeatures& features, - Primitive::Type type, - bool is_volatile, - HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::NoLocation()); // Unused receiver. - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetInAt(3, Location::RequiresRegister()); - - if (type == Primitive::kPrimLong) { - // Potentially need temps for ldrexd-strexd loop. - if (is_volatile && !features.HasAtomicLdrdAndStrd()) { - locations->AddTemp(Location::RequiresRegister()); // Temp_lo. - locations->AddTemp(Location::RequiresRegister()); // Temp_hi. - } - } else if (type == Primitive::kPrimNot) { - // Temps for card-marking. - locations->AddTemp(Location::RequiresRegister()); // Temp. - locations->AddTemp(Location::RequiresRegister()); // Card. - } -} - -void IntrinsicLocationsBuilderARM::VisitUnsafePut(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutOrdered(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutVolatile(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ true, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutObject(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ true, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutLong(HInvoke* invoke) { - CreateIntIntIntIntToVoid( - arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutLongOrdered(HInvoke* invoke) { - CreateIntIntIntIntToVoid( - arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutLongVolatile(HInvoke* invoke) { - CreateIntIntIntIntToVoid( - arena_, features_, Primitive::kPrimLong, /* is_volatile */ true, invoke); -} - -static void GenUnsafePut(LocationSummary* locations, - Primitive::Type type, - bool is_volatile, - bool is_ordered, - CodeGeneratorARM* codegen) { - ArmAssembler* assembler = codegen->GetAssembler(); - - Register base = locations->InAt(1).AsRegister<Register>(); // Object pointer. - Register offset = locations->InAt(2).AsRegisterPairLow<Register>(); // Long offset, lo part only. - Register value; - - if (is_volatile || is_ordered) { - __ dmb(ISH); - } - - if (type == Primitive::kPrimLong) { - Register value_lo = locations->InAt(3).AsRegisterPairLow<Register>(); - value = value_lo; - if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { - Register temp_lo = locations->GetTemp(0).AsRegister<Register>(); - Register temp_hi = locations->GetTemp(1).AsRegister<Register>(); - Register value_hi = locations->InAt(3).AsRegisterPairHigh<Register>(); - - __ add(IP, base, ShifterOperand(offset)); - Label loop_head; - __ Bind(&loop_head); - __ ldrexd(temp_lo, temp_hi, IP); - __ strexd(temp_lo, value_lo, value_hi, IP); - __ cmp(temp_lo, ShifterOperand(0)); - __ b(&loop_head, NE); - } else { - __ add(IP, base, ShifterOperand(offset)); - __ strd(value_lo, Address(IP)); - } - } else { - value = locations->InAt(3).AsRegister<Register>(); - Register source = value; - if (kPoisonHeapReferences && type == Primitive::kPrimNot) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - __ Mov(temp, value); - __ PoisonHeapReference(temp); - source = temp; - } - __ str(source, Address(base, offset)); - } - - if (is_volatile) { - __ dmb(ISH); - } - - if (type == Primitive::kPrimNot) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register card = locations->GetTemp(1).AsRegister<Register>(); - bool value_can_be_null = true; // TODO: Worth finding out this information? - codegen->MarkGCCard(temp, card, base, value, value_can_be_null); - } -} - -void IntrinsicCodeGeneratorARM::VisitUnsafePut(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimInt, - /* is_volatile */ false, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimInt, - /* is_volatile */ false, - /* is_ordered */ true, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimInt, - /* is_volatile */ true, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutObject(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimNot, - /* is_volatile */ false, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimNot, - /* is_volatile */ false, - /* is_ordered */ true, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimNot, - /* is_volatile */ true, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutLong(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimLong, - /* is_volatile */ false, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutLongOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimLong, - /* is_volatile */ false, - /* is_ordered */ true, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutLongVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimLong, - /* is_volatile */ true, - /* is_ordered */ false, - codegen_); -} - -static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena, - HInvoke* invoke, - Primitive::Type type) { - bool can_call = kEmitCompilerReadBarrier && - kUseBakerReadBarrier && - (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject); - LocationSummary* locations = new (arena) LocationSummary(invoke, - (can_call - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall), - kIntrinsified); - locations->SetInAt(0, Location::NoLocation()); // Unused receiver. - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetInAt(3, Location::RequiresRegister()); - locations->SetInAt(4, Location::RequiresRegister()); - - // If heap poisoning is enabled, we don't want the unpoisoning - // operations to potentially clobber the output. Likewise when - // emitting a (Baker) read barrier, which may call. - Location::OutputOverlap overlaps = - ((kPoisonHeapReferences && type == Primitive::kPrimNot) || can_call) - ? Location::kOutputOverlap - : Location::kNoOutputOverlap; - locations->SetOut(Location::RequiresRegister(), overlaps); - - // Temporary registers used in CAS. In the object case - // (UnsafeCASObject intrinsic), these are also used for - // card-marking, and possibly for (Baker) read barrier. - locations->AddTemp(Location::RequiresRegister()); // Pointer. - locations->AddTemp(Location::RequiresRegister()); // Temp 1. -} - -static void GenCas(HInvoke* invoke, Primitive::Type type, CodeGeneratorARM* codegen) { - DCHECK_NE(type, Primitive::kPrimLong); - - ArmAssembler* assembler = codegen->GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); // Boolean result. - - Register base = locations->InAt(1).AsRegister<Register>(); // Object pointer. - Location offset_loc = locations->InAt(2); - Register offset = offset_loc.AsRegisterPairLow<Register>(); // Offset (discard high 4B). - Register expected = locations->InAt(3).AsRegister<Register>(); // Expected. - Register value = locations->InAt(4).AsRegister<Register>(); // Value. - - Location tmp_ptr_loc = locations->GetTemp(0); - Register tmp_ptr = tmp_ptr_loc.AsRegister<Register>(); // Pointer to actual memory. - Register tmp = locations->GetTemp(1).AsRegister<Register>(); // Value in memory. - - if (type == Primitive::kPrimNot) { - // The only read barrier implementation supporting the - // UnsafeCASObject intrinsic is the Baker-style read barriers. - DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); - - // Mark card for object assuming new value is stored. Worst case we will mark an unchanged - // object and scan the receiver at the next GC for nothing. - bool value_can_be_null = true; // TODO: Worth finding out this information? - codegen->MarkGCCard(tmp_ptr, tmp, base, value, value_can_be_null); - - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // Need to make sure the reference stored in the field is a to-space - // one before attempting the CAS or the CAS could fail incorrectly. - codegen->UpdateReferenceFieldWithBakerReadBarrier( - invoke, - out_loc, // Unused, used only as a "temporary" within the read barrier. - base, - /* field_offset */ offset_loc, - tmp_ptr_loc, - /* needs_null_check */ false, - tmp); - } - } - - // Prevent reordering with prior memory operations. - // Emit a DMB ISH instruction instead of an DMB ISHST one, as the - // latter allows a preceding load to be delayed past the STXR - // instruction below. - __ dmb(ISH); - - __ add(tmp_ptr, base, ShifterOperand(offset)); - - if (kPoisonHeapReferences && type == Primitive::kPrimNot) { - __ PoisonHeapReference(expected); - if (value == expected) { - // Do not poison `value`, as it is the same register as - // `expected`, which has just been poisoned. - } else { - __ PoisonHeapReference(value); - } - } - - // do { - // tmp = [r_ptr] - expected; - // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); - // result = tmp != 0; - - Label loop_head; - __ Bind(&loop_head); - - __ ldrex(tmp, tmp_ptr); - - __ subs(tmp, tmp, ShifterOperand(expected)); - - __ it(EQ, ItState::kItT); - __ strex(tmp, value, tmp_ptr, EQ); - __ cmp(tmp, ShifterOperand(1), EQ); - - __ b(&loop_head, EQ); - - __ dmb(ISH); - - __ rsbs(out, tmp, ShifterOperand(1)); - __ it(CC); - __ mov(out, ShifterOperand(0), CC); - - if (kPoisonHeapReferences && type == Primitive::kPrimNot) { - __ UnpoisonHeapReference(expected); - if (value == expected) { - // Do not unpoison `value`, as it is the same register as - // `expected`, which has just been unpoisoned. - } else { - __ UnpoisonHeapReference(value); - } - } -} - -void IntrinsicLocationsBuilderARM::VisitUnsafeCASInt(HInvoke* invoke) { - CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimInt); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeCASObject(HInvoke* invoke) { - // The only read barrier implementation supporting the - // UnsafeCASObject intrinsic is the Baker-style read barriers. - if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { - return; - } - - CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimNot); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeCASInt(HInvoke* invoke) { - GenCas(invoke, Primitive::kPrimInt, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeCASObject(HInvoke* invoke) { - // The only read barrier implementation supporting the - // UnsafeCASObject intrinsic is the Baker-style read barriers. - DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); - - GenCas(invoke, Primitive::kPrimNot, codegen_); -} - -void IntrinsicLocationsBuilderARM::VisitStringCompareTo(HInvoke* invoke) { - // The inputs plus one temp. - LocationSummary* locations = new (arena_) LocationSummary(invoke, - invoke->InputAt(1)->CanBeNull() - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - // Need temporary registers for String compression's feature. - if (mirror::kUseStringCompression) { - locations->AddTemp(Location::RequiresRegister()); - } - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitStringCompareTo(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register str = locations->InAt(0).AsRegister<Register>(); - Register arg = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - Register temp0 = locations->GetTemp(0).AsRegister<Register>(); - Register temp1 = locations->GetTemp(1).AsRegister<Register>(); - Register temp2 = locations->GetTemp(2).AsRegister<Register>(); - Register temp3; - if (mirror::kUseStringCompression) { - temp3 = locations->GetTemp(3).AsRegister<Register>(); - } - - Label loop; - Label find_char_diff; - Label end; - Label different_compression; - - // Get offsets of count and value fields within a string object. - const int32_t count_offset = mirror::String::CountOffset().Int32Value(); - const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); - - // Note that the null check must have been done earlier. - DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); - - // Take slow path and throw if input can be and is null. - SlowPathCode* slow_path = nullptr; - const bool can_slow_path = invoke->InputAt(1)->CanBeNull(); - if (can_slow_path) { - slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(slow_path); - __ CompareAndBranchIfZero(arg, slow_path->GetEntryLabel()); - } - - // Reference equality check, return 0 if same reference. - __ subs(out, str, ShifterOperand(arg)); - __ b(&end, EQ); - - if (mirror::kUseStringCompression) { - // Load `count` fields of this and argument strings. - __ ldr(temp3, Address(str, count_offset)); - __ ldr(temp2, Address(arg, count_offset)); - // Extract lengths from the `count` fields. - __ Lsr(temp0, temp3, 1u); - __ Lsr(temp1, temp2, 1u); - } else { - // Load lengths of this and argument strings. - __ ldr(temp0, Address(str, count_offset)); - __ ldr(temp1, Address(arg, count_offset)); - } - // out = length diff. - __ subs(out, temp0, ShifterOperand(temp1)); - // temp0 = min(len(str), len(arg)). - __ it(GT); - __ mov(temp0, ShifterOperand(temp1), GT); - // Shorter string is empty? - __ CompareAndBranchIfZero(temp0, &end); - - if (mirror::kUseStringCompression) { - // Check if both strings using same compression style to use this comparison loop. - __ eor(temp2, temp2, ShifterOperand(temp3)); - __ Lsrs(temp2, temp2, 1u); - __ b(&different_compression, CS); - // For string compression, calculate the number of bytes to compare (not chars). - // This could in theory exceed INT32_MAX, so treat temp0 as unsigned. - __ Lsls(temp3, temp3, 31u); // Extract purely the compression flag. - __ it(NE); - __ add(temp0, temp0, ShifterOperand(temp0), NE); - } - - // Store offset of string value in preparation for comparison loop. - __ mov(temp1, ShifterOperand(value_offset)); - - // Assertions that must hold in order to compare multiple characters at a time. - CHECK_ALIGNED(value_offset, 8); - static_assert(IsAligned<8>(kObjectAlignment), - "String data must be 8-byte aligned for unrolled CompareTo loop."); - - const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); - DCHECK_EQ(char_size, 2u); - - Label find_char_diff_2nd_cmp; - // Unrolled loop comparing 4x16-bit chars per iteration (ok because of string data alignment). - __ Bind(&loop); - __ ldr(IP, Address(str, temp1)); - __ ldr(temp2, Address(arg, temp1)); - __ cmp(IP, ShifterOperand(temp2)); - __ b(&find_char_diff, NE); - __ add(temp1, temp1, ShifterOperand(char_size * 2)); - - __ ldr(IP, Address(str, temp1)); - __ ldr(temp2, Address(arg, temp1)); - __ cmp(IP, ShifterOperand(temp2)); - __ b(&find_char_diff_2nd_cmp, NE); - __ add(temp1, temp1, ShifterOperand(char_size * 2)); - // With string compression, we have compared 8 bytes, otherwise 4 chars. - __ subs(temp0, temp0, ShifterOperand(mirror::kUseStringCompression ? 8 : 4)); - __ b(&loop, HI); - __ b(&end); - - __ Bind(&find_char_diff_2nd_cmp); - if (mirror::kUseStringCompression) { - __ subs(temp0, temp0, ShifterOperand(4)); // 4 bytes previously compared. - __ b(&end, LS); // Was the second comparison fully beyond the end? - } else { - // Without string compression, we can start treating temp0 as signed - // and rely on the signed comparison below. - __ sub(temp0, temp0, ShifterOperand(2)); - } - - // Find the single character difference. - __ Bind(&find_char_diff); - // Get the bit position of the first character that differs. - __ eor(temp1, temp2, ShifterOperand(IP)); - __ rbit(temp1, temp1); - __ clz(temp1, temp1); - - // temp0 = number of characters remaining to compare. - // (Without string compression, it could be < 1 if a difference is found by the second CMP - // in the comparison loop, and after the end of the shorter string data). - - // Without string compression (temp1 >> 4) = character where difference occurs between the last - // two words compared, in the interval [0,1]. - // (0 for low half-word different, 1 for high half-word different). - // With string compression, (temp1 << 3) = byte where the difference occurs, - // in the interval [0,3]. - - // If temp0 <= (temp1 >> (kUseStringCompression ? 3 : 4)), the difference occurs outside - // the remaining string data, so just return length diff (out). - // The comparison is unsigned for string compression, otherwise signed. - __ cmp(temp0, ShifterOperand(temp1, LSR, mirror::kUseStringCompression ? 3 : 4)); - __ b(&end, mirror::kUseStringCompression ? LS : LE); - - // Extract the characters and calculate the difference. - if (mirror::kUseStringCompression) { - // For compressed strings we need to clear 0x7 from temp1, for uncompressed we need to clear - // 0xf. We also need to prepare the character extraction mask `uncompressed ? 0xffffu : 0xffu`. - // The compression flag is now in the highest bit of temp3, so let's play some tricks. - __ orr(temp3, temp3, ShifterOperand(0xffu << 23)); // uncompressed ? 0xff800000u : 0x7ff80000u - __ bic(temp1, temp1, ShifterOperand(temp3, LSR, 31 - 3)); // &= ~(uncompressed ? 0xfu : 0x7u) - __ Asr(temp3, temp3, 7u); // uncompressed ? 0xffff0000u : 0xff0000u. - __ Lsr(temp2, temp2, temp1); // Extract second character. - __ Lsr(temp3, temp3, 16u); // uncompressed ? 0xffffu : 0xffu - __ Lsr(out, IP, temp1); // Extract first character. - __ and_(temp2, temp2, ShifterOperand(temp3)); - __ and_(out, out, ShifterOperand(temp3)); - } else { - __ bic(temp1, temp1, ShifterOperand(0xf)); - __ Lsr(temp2, temp2, temp1); - __ Lsr(out, IP, temp1); - __ movt(temp2, 0); - __ movt(out, 0); - } - - __ sub(out, out, ShifterOperand(temp2)); - - if (mirror::kUseStringCompression) { - __ b(&end); - __ Bind(&different_compression); - - // Comparison for different compression style. - const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte); - DCHECK_EQ(c_char_size, 1u); - - // We want to free up the temp3, currently holding `str.count`, for comparison. - // So, we move it to the bottom bit of the iteration count `temp0` which we tnen - // need to treat as unsigned. Start by freeing the bit with an ADD and continue - // further down by a LSRS+SBC which will flip the meaning of the flag but allow - // `subs temp0, #2; bhi different_compression_loop` to serve as the loop condition. - __ add(temp0, temp0, ShifterOperand(temp0)); // Unlike LSL, this ADD is always 16-bit. - // `temp1` will hold the compressed data pointer, `temp2` the uncompressed data pointer. - __ mov(temp1, ShifterOperand(str)); - __ mov(temp2, ShifterOperand(arg)); - __ Lsrs(temp3, temp3, 1u); // Continue the move of the compression flag. - __ it(CS, kItThen); // Interleave with selection of temp1 and temp2. - __ mov(temp1, ShifterOperand(arg), CS); // Preserves flags. - __ mov(temp2, ShifterOperand(str), CS); // Preserves flags. - __ sbc(temp0, temp0, ShifterOperand(0)); // Complete the move of the compression flag. - - // Adjust temp1 and temp2 from string pointers to data pointers. - __ add(temp1, temp1, ShifterOperand(value_offset)); - __ add(temp2, temp2, ShifterOperand(value_offset)); - - Label different_compression_loop; - Label different_compression_diff; - - // Main loop for different compression. - __ Bind(&different_compression_loop); - __ ldrb(IP, Address(temp1, c_char_size, Address::PostIndex)); - __ ldrh(temp3, Address(temp2, char_size, Address::PostIndex)); - __ cmp(IP, ShifterOperand(temp3)); - __ b(&different_compression_diff, NE); - __ subs(temp0, temp0, ShifterOperand(2)); - __ b(&different_compression_loop, HI); - __ b(&end); - - // Calculate the difference. - __ Bind(&different_compression_diff); - __ sub(out, IP, ShifterOperand(temp3)); - // Flip the difference if the `arg` is compressed. - // `temp0` contains inverted `str` compression flag, i.e the same as `arg` compression flag. - __ Lsrs(temp0, temp0, 1u); - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ it(CC); - __ rsb(out, out, ShifterOperand(0), CC); - } - - __ Bind(&end); - - if (can_slow_path) { - __ Bind(slow_path->GetExitLabel()); - } -} - -void IntrinsicLocationsBuilderARM::VisitStringEquals(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - // Temporary registers to store lengths of strings and for calculations. - // Using instruction cbz requires a low register, so explicitly set a temp to be R0. - locations->AddTemp(Location::RegisterLocation(R0)); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - - locations->SetOut(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitStringEquals(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register str = locations->InAt(0).AsRegister<Register>(); - Register arg = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register temp1 = locations->GetTemp(1).AsRegister<Register>(); - Register temp2 = locations->GetTemp(2).AsRegister<Register>(); - - Label loop; - Label end; - Label return_true; - Label return_false; - Label* final_label = codegen_->GetFinalLabel(invoke, &end); - - // Get offsets of count, value, and class fields within a string object. - const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); - const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); - const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value(); - - // Note that the null check must have been done earlier. - DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); - - StringEqualsOptimizations optimizations(invoke); - if (!optimizations.GetArgumentNotNull()) { - // Check if input is null, return false if it is. - __ CompareAndBranchIfZero(arg, &return_false); - } - - // Reference equality check, return true if same reference. - __ cmp(str, ShifterOperand(arg)); - __ b(&return_true, EQ); - - if (!optimizations.GetArgumentIsString()) { - // Instanceof check for the argument by comparing class fields. - // All string objects must have the same type since String cannot be subclassed. - // Receiver must be a string object, so its class field is equal to all strings' class fields. - // If the argument is a string object, its class field must be equal to receiver's class field. - __ ldr(temp, Address(str, class_offset)); - __ ldr(temp1, Address(arg, class_offset)); - __ cmp(temp, ShifterOperand(temp1)); - __ b(&return_false, NE); - } - - // Load `count` fields of this and argument strings. - __ ldr(temp, Address(str, count_offset)); - __ ldr(temp1, Address(arg, count_offset)); - // Check if `count` fields are equal, return false if they're not. - // Also compares the compression style, if differs return false. - __ cmp(temp, ShifterOperand(temp1)); - __ b(&return_false, NE); - // Return true if both strings are empty. Even with string compression `count == 0` means empty. - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ cbz(temp, &return_true); - - // Assertions that must hold in order to compare strings 4 bytes at a time. - DCHECK_ALIGNED(value_offset, 4); - static_assert(IsAligned<4>(kObjectAlignment), "String data must be aligned for fast compare."); - - if (mirror::kUseStringCompression) { - // For string compression, calculate the number of bytes to compare (not chars). - // This could in theory exceed INT32_MAX, so treat temp as unsigned. - __ Lsrs(temp, temp, 1u); // Extract length and check compression flag. - __ it(CS); // If uncompressed, - __ add(temp, temp, ShifterOperand(temp), CS); // double the byte count. - } - - // Store offset of string value in preparation for comparison loop. - __ LoadImmediate(temp1, value_offset); - - // Loop to compare strings 4 bytes at a time starting at the front of the string. - // Ok to do this because strings are zero-padded to kObjectAlignment. - __ Bind(&loop); - __ ldr(out, Address(str, temp1)); - __ ldr(temp2, Address(arg, temp1)); - __ add(temp1, temp1, ShifterOperand(sizeof(uint32_t))); - __ cmp(out, ShifterOperand(temp2)); - __ b(&return_false, NE); - // With string compression, we have compared 4 bytes, otherwise 2 chars. - __ subs(temp, temp, ShifterOperand(mirror::kUseStringCompression ? 4 : 2)); - __ b(&loop, HI); - - // Return true and exit the function. - // If loop does not result in returning false, we return true. - __ Bind(&return_true); - __ LoadImmediate(out, 1); - __ b(final_label); - - // Return false and exit the function. - __ Bind(&return_false); - __ LoadImmediate(out, 0); - - if (end.IsLinked()) { - __ Bind(&end); - } -} - -static void GenerateVisitStringIndexOf(HInvoke* invoke, - ArmAssembler* assembler, - CodeGeneratorARM* codegen, - ArenaAllocator* allocator, - bool start_at_zero) { - LocationSummary* locations = invoke->GetLocations(); - - // Note that the null check must have been done earlier. - DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); - - // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, - // or directly dispatch for a large constant, or omit slow-path for a small constant or a char. - SlowPathCode* slow_path = nullptr; - HInstruction* code_point = invoke->InputAt(1); - if (code_point->IsIntConstant()) { - if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > - std::numeric_limits<uint16_t>::max()) { - // Always needs the slow-path. We could directly dispatch to it, but this case should be - // rare, so for simplicity just put the full slow-path down and branch unconditionally. - slow_path = new (allocator) IntrinsicSlowPathARM(invoke); - codegen->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); - return; - } - } else if (code_point->GetType() != Primitive::kPrimChar) { - Register char_reg = locations->InAt(1).AsRegister<Register>(); - // 0xffff is not modified immediate but 0x10000 is, so use `>= 0x10000` instead of `> 0xffff`. - __ cmp(char_reg, - ShifterOperand(static_cast<uint32_t>(std::numeric_limits<uint16_t>::max()) + 1)); - slow_path = new (allocator) IntrinsicSlowPathARM(invoke); - codegen->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), HS); - } - - if (start_at_zero) { - Register tmp_reg = locations->GetTemp(0).AsRegister<Register>(); - DCHECK_EQ(tmp_reg, R2); - // Start-index = 0. - __ LoadImmediate(tmp_reg, 0); - } - - codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path); - CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>(); - - if (slow_path != nullptr) { - __ Bind(slow_path->GetExitLabel()); - } -} - -void IntrinsicLocationsBuilderARM::VisitStringIndexOf(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's - // best to align the inputs accordingly. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetOut(Location::RegisterLocation(R0)); - - // Need to send start-index=0. - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); -} - -void IntrinsicCodeGeneratorARM::VisitStringIndexOf(HInvoke* invoke) { - GenerateVisitStringIndexOf( - invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); -} - -void IntrinsicLocationsBuilderARM::VisitStringIndexOfAfter(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's - // best to align the inputs accordingly. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringIndexOfAfter(HInvoke* invoke) { - GenerateVisitStringIndexOf( - invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); -} - -void IntrinsicLocationsBuilderARM::VisitStringNewStringFromBytes(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->SetInAt(3, Location::RegisterLocation(calling_convention.GetRegisterAt(3))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringNewStringFromBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register byte_array = locations->InAt(0).AsRegister<Register>(); - __ cmp(byte_array, ShifterOperand(0)); - SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), EQ); - - codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc(), slow_path); - CheckEntrypointTypes<kQuickAllocStringFromBytes, void*, void*, int32_t, int32_t, int32_t>(); - __ Bind(slow_path->GetExitLabel()); -} - -void IntrinsicLocationsBuilderARM::VisitStringNewStringFromChars(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainOnly, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringNewStringFromChars(HInvoke* invoke) { - // No need to emit code checking whether `locations->InAt(2)` is a null - // pointer, as callers of the native method - // - // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) - // - // all include a null check on `data` before calling that method. - codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); - CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>(); -} - -void IntrinsicLocationsBuilderARM::VisitStringNewStringFromString(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringNewStringFromString(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register string_to_copy = locations->InAt(0).AsRegister<Register>(); - __ cmp(string_to_copy, ShifterOperand(0)); - SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), EQ); - - codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc(), slow_path); - CheckEntrypointTypes<kQuickAllocStringFromString, void*, void*>(); - - __ Bind(slow_path->GetExitLabel()); -} - -void IntrinsicLocationsBuilderARM::VisitSystemArrayCopy(HInvoke* invoke) { - // The only read barrier implementation supporting the - // SystemArrayCopy intrinsic is the Baker-style read barriers. - if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { - return; - } - - CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke); - LocationSummary* locations = invoke->GetLocations(); - if (locations == nullptr) { - return; - } - - HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); - HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); - HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); - - if (src_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(src_pos->GetValue())) { - locations->SetInAt(1, Location::RequiresRegister()); - } - if (dest_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(dest_pos->GetValue())) { - locations->SetInAt(3, Location::RequiresRegister()); - } - if (length != nullptr && !assembler_->ShifterOperandCanAlwaysHold(length->GetValue())) { - locations->SetInAt(4, Location::RequiresRegister()); - } - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // Temporary register IP cannot be used in - // ReadBarrierSystemArrayCopySlowPathARM (because that register - // is clobbered by ReadBarrierMarkRegX entry points). Get an extra - // temporary register from the register allocator. - locations->AddTemp(Location::RequiresRegister()); - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen_); - arm_codegen->MaybeAddBakerCcEntrypointTempForFields(locations); - } -} - -static void CheckPosition(ArmAssembler* assembler, - Location pos, - Register input, - Location length, - SlowPathCode* slow_path, - Register temp, - bool length_is_input_length = false) { - // Where is the length in the Array? - const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value(); - - if (pos.IsConstant()) { - int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); - if (pos_const == 0) { - if (!length_is_input_length) { - // Check that length(input) >= length. - __ LoadFromOffset(kLoadWord, temp, input, length_offset); - if (length.IsConstant()) { - __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); - } else { - __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); - } - __ b(slow_path->GetEntryLabel(), LT); - } - } else { - // Check that length(input) >= pos. - __ LoadFromOffset(kLoadWord, temp, input, length_offset); - __ subs(temp, temp, ShifterOperand(pos_const)); - __ b(slow_path->GetEntryLabel(), LT); - - // Check that (length(input) - pos) >= length. - if (length.IsConstant()) { - __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); - } else { - __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); - } - __ b(slow_path->GetEntryLabel(), LT); - } - } else if (length_is_input_length) { - // The only way the copy can succeed is if pos is zero. - Register pos_reg = pos.AsRegister<Register>(); - __ CompareAndBranchIfNonZero(pos_reg, slow_path->GetEntryLabel()); - } else { - // Check that pos >= 0. - Register pos_reg = pos.AsRegister<Register>(); - __ cmp(pos_reg, ShifterOperand(0)); - __ b(slow_path->GetEntryLabel(), LT); - - // Check that pos <= length(input). - __ LoadFromOffset(kLoadWord, temp, input, length_offset); - __ subs(temp, temp, ShifterOperand(pos_reg)); - __ b(slow_path->GetEntryLabel(), LT); - - // Check that (length(input) - pos) >= length. - if (length.IsConstant()) { - __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); - } else { - __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); - } - __ b(slow_path->GetEntryLabel(), LT); - } -} - -void IntrinsicCodeGeneratorARM::VisitSystemArrayCopy(HInvoke* invoke) { - // The only read barrier implementation supporting the - // SystemArrayCopy intrinsic is the Baker-style read barriers. - DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); - - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); - - Register src = locations->InAt(0).AsRegister<Register>(); - Location src_pos = locations->InAt(1); - Register dest = locations->InAt(2).AsRegister<Register>(); - Location dest_pos = locations->InAt(3); - Location length = locations->InAt(4); - Location temp1_loc = locations->GetTemp(0); - Register temp1 = temp1_loc.AsRegister<Register>(); - Location temp2_loc = locations->GetTemp(1); - Register temp2 = temp2_loc.AsRegister<Register>(); - Location temp3_loc = locations->GetTemp(2); - Register temp3 = temp3_loc.AsRegister<Register>(); - - SlowPathCode* intrinsic_slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(intrinsic_slow_path); - - Label conditions_on_positions_validated; - SystemArrayCopyOptimizations optimizations(invoke); - - // If source and destination are the same, we go to slow path if we need to do - // forward copying. - if (src_pos.IsConstant()) { - int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); - if (dest_pos.IsConstant()) { - int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); - if (optimizations.GetDestinationIsSource()) { - // Checked when building locations. - DCHECK_GE(src_pos_constant, dest_pos_constant); - } else if (src_pos_constant < dest_pos_constant) { - __ cmp(src, ShifterOperand(dest)); - __ b(intrinsic_slow_path->GetEntryLabel(), EQ); - } - - // Checked when building locations. - DCHECK(!optimizations.GetDestinationIsSource() - || (src_pos_constant >= dest_pos.GetConstant()->AsIntConstant()->GetValue())); - } else { - if (!optimizations.GetDestinationIsSource()) { - __ cmp(src, ShifterOperand(dest)); - __ b(&conditions_on_positions_validated, NE); - } - __ cmp(dest_pos.AsRegister<Register>(), ShifterOperand(src_pos_constant)); - __ b(intrinsic_slow_path->GetEntryLabel(), GT); - } - } else { - if (!optimizations.GetDestinationIsSource()) { - __ cmp(src, ShifterOperand(dest)); - __ b(&conditions_on_positions_validated, NE); - } - if (dest_pos.IsConstant()) { - int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); - __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos_constant)); - } else { - __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos.AsRegister<Register>())); - } - __ b(intrinsic_slow_path->GetEntryLabel(), LT); - } - - __ Bind(&conditions_on_positions_validated); - - if (!optimizations.GetSourceIsNotNull()) { - // Bail out if the source is null. - __ CompareAndBranchIfZero(src, intrinsic_slow_path->GetEntryLabel()); - } - - if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { - // Bail out if the destination is null. - __ CompareAndBranchIfZero(dest, intrinsic_slow_path->GetEntryLabel()); - } - - // If the length is negative, bail out. - // We have already checked in the LocationsBuilder for the constant case. - if (!length.IsConstant() && - !optimizations.GetCountIsSourceLength() && - !optimizations.GetCountIsDestinationLength()) { - __ cmp(length.AsRegister<Register>(), ShifterOperand(0)); - __ b(intrinsic_slow_path->GetEntryLabel(), LT); - } - - // Validity checks: source. - CheckPosition(assembler, - src_pos, - src, - length, - intrinsic_slow_path, - temp1, - optimizations.GetCountIsSourceLength()); - - // Validity checks: dest. - CheckPosition(assembler, - dest_pos, - dest, - length, - intrinsic_slow_path, - temp1, - optimizations.GetCountIsDestinationLength()); - - if (!optimizations.GetDoesNotNeedTypeCheck()) { - // Check whether all elements of the source array are assignable to the component - // type of the destination array. We do two checks: the classes are the same, - // or the destination is Object[]. If none of these checks succeed, we go to the - // slow path. - - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - if (!optimizations.GetSourceIsNonPrimitiveArray()) { - // /* HeapReference<Class> */ temp1 = src->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, src, class_offset, temp2_loc, /* needs_null_check */ false); - // Bail out if the source is not a non primitive array. - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); - __ CompareAndBranchIfZero(temp1, intrinsic_slow_path->GetEntryLabel()); - // If heap poisoning is enabled, `temp1` has been unpoisoned - // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. - // /* uint16_t */ temp1 = static_cast<uint16>(temp1->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp1, temp1, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); - } - - // /* HeapReference<Class> */ temp1 = dest->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, dest, class_offset, temp2_loc, /* needs_null_check */ false); - - if (!optimizations.GetDestinationIsNonPrimitiveArray()) { - // Bail out if the destination is not a non primitive array. - // - // Register `temp1` is not trashed by the read barrier emitted - // by GenerateFieldLoadWithBakerReadBarrier below, as that - // method produces a call to a ReadBarrierMarkRegX entry point, - // which saves all potentially live registers, including - // temporaries such a `temp1`. - // /* HeapReference<Class> */ temp2 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp2_loc, temp1, component_offset, temp3_loc, /* needs_null_check */ false); - __ CompareAndBranchIfZero(temp2, intrinsic_slow_path->GetEntryLabel()); - // If heap poisoning is enabled, `temp2` has been unpoisoned - // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. - // /* uint16_t */ temp2 = static_cast<uint16>(temp2->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp2, temp2, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp2, intrinsic_slow_path->GetEntryLabel()); - } - - // For the same reason given earlier, `temp1` is not trashed by the - // read barrier emitted by GenerateFieldLoadWithBakerReadBarrier below. - // /* HeapReference<Class> */ temp2 = src->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp2_loc, src, class_offset, temp3_loc, /* needs_null_check */ false); - // Note: if heap poisoning is on, we are comparing two unpoisoned references here. - __ cmp(temp1, ShifterOperand(temp2)); - - if (optimizations.GetDestinationIsTypedObjectArray()) { - Label do_copy; - __ b(&do_copy, EQ); - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); - // /* HeapReference<Class> */ temp1 = temp1->super_class_ - // We do not need to emit a read barrier for the following - // heap reference load, as `temp1` is only used in a - // comparison with null below, and this reference is not - // kept afterwards. - __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); - __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); - __ Bind(&do_copy); - } else { - __ b(intrinsic_slow_path->GetEntryLabel(), NE); - } - } else { - // Non read barrier code. - - // /* HeapReference<Class> */ temp1 = dest->klass_ - __ LoadFromOffset(kLoadWord, temp1, dest, class_offset); - // /* HeapReference<Class> */ temp2 = src->klass_ - __ LoadFromOffset(kLoadWord, temp2, src, class_offset); - bool did_unpoison = false; - if (!optimizations.GetDestinationIsNonPrimitiveArray() || - !optimizations.GetSourceIsNonPrimitiveArray()) { - // One or two of the references need to be unpoisoned. Unpoison them - // both to make the identity check valid. - __ MaybeUnpoisonHeapReference(temp1); - __ MaybeUnpoisonHeapReference(temp2); - did_unpoison = true; - } - - if (!optimizations.GetDestinationIsNonPrimitiveArray()) { - // Bail out if the destination is not a non primitive array. - // /* HeapReference<Class> */ temp3 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - __ MaybeUnpoisonHeapReference(temp3); - // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); - } - - if (!optimizations.GetSourceIsNonPrimitiveArray()) { - // Bail out if the source is not a non primitive array. - // /* HeapReference<Class> */ temp3 = temp2->component_type_ - __ LoadFromOffset(kLoadWord, temp3, temp2, component_offset); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - __ MaybeUnpoisonHeapReference(temp3); - // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); - } - - __ cmp(temp1, ShifterOperand(temp2)); - - if (optimizations.GetDestinationIsTypedObjectArray()) { - Label do_copy; - __ b(&do_copy, EQ); - if (!did_unpoison) { - __ MaybeUnpoisonHeapReference(temp1); - } - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); - __ MaybeUnpoisonHeapReference(temp1); - // /* HeapReference<Class> */ temp1 = temp1->super_class_ - __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); - // No need to unpoison the result, we're comparing against null. - __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); - __ Bind(&do_copy); - } else { - __ b(intrinsic_slow_path->GetEntryLabel(), NE); - } - } - } else if (!optimizations.GetSourceIsNonPrimitiveArray()) { - DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); - // Bail out if the source is not a non primitive array. - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // /* HeapReference<Class> */ temp1 = src->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, src, class_offset, temp2_loc, /* needs_null_check */ false); - // /* HeapReference<Class> */ temp3 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp3_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - // If heap poisoning is enabled, `temp3` has been unpoisoned - // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. - } else { - // /* HeapReference<Class> */ temp1 = src->klass_ - __ LoadFromOffset(kLoadWord, temp1, src, class_offset); - __ MaybeUnpoisonHeapReference(temp1); - // /* HeapReference<Class> */ temp3 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - __ MaybeUnpoisonHeapReference(temp3); - } - // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); - } - - if (length.IsConstant() && length.GetConstant()->AsIntConstant()->GetValue() == 0) { - // Null constant length: not need to emit the loop code at all. - } else { - Label done; - const Primitive::Type type = Primitive::kPrimNot; - const int32_t element_size = Primitive::ComponentSize(type); - - if (length.IsRegister()) { - // Don't enter the copy loop if the length is null. - __ CompareAndBranchIfZero(length.AsRegister<Register>(), &done); - } - - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // TODO: Also convert this intrinsic to the IsGcMarking strategy? - - // SystemArrayCopy implementation for Baker read barriers (see - // also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier): - // - // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // // Slow-path copy. - // do { - // *dest_ptr++ = MaybePoison(ReadBarrier::Mark(MaybeUnpoison(*src_ptr++))); - // } while (src_ptr != end_ptr) - // } else { - // // Fast-path copy. - // do { - // *dest_ptr++ = *src_ptr++; - // } while (src_ptr != end_ptr) - // } - - // /* int32_t */ monitor = src->monitor_ - __ LoadFromOffset(kLoadWord, temp2, src, monitor_offset); - // /* LockWord */ lock_word = LockWord(monitor) - static_assert(sizeof(LockWord) == sizeof(int32_t), - "art::LockWord and int32_t have different sizes."); - - // Introduce a dependency on the lock_word including the rb_state, - // which shall prevent load-load reordering without using - // a memory barrier (which would be more expensive). - // `src` is unchanged by this operation, but its value now depends - // on `temp2`. - __ add(src, src, ShifterOperand(temp2, LSR, 32)); - - // Compute the base source address in `temp1`. - // Note that `temp1` (the base source address) is computed from - // `src` (and `src_pos`) here, and thus honors the artificial - // dependency of `src` on `temp2`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1); - // Compute the end source address in `temp3`. - GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3); - // The base destination address is computed later, as `temp2` is - // used for intermediate computations. - - // Slow path used to copy array when `src` is gray. - // Note that the base destination address is computed in `temp2` - // by the slow path code. - SlowPathCode* read_barrier_slow_path = - new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathARM(invoke); - codegen_->AddSlowPath(read_barrier_slow_path); - - // Given the numeric representation, it's enough to check the low bit of the - // rb_state. We do that by shifting the bit out of the lock word with LSRS - // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Lsrs(temp2, temp2, LockWord::kReadBarrierStateShift + 1); - // Carry flag is the last bit shifted out by LSRS. - __ b(read_barrier_slow_path->GetEntryLabel(), CS); - - // Fast-path copy. - // Compute the base destination address in `temp2`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2); - // Iterate over the arrays and do a raw copy of the objects. We don't need to - // poison/unpoison. - Label loop; - __ Bind(&loop); - __ ldr(IP, Address(temp1, element_size, Address::PostIndex)); - __ str(IP, Address(temp2, element_size, Address::PostIndex)); - __ cmp(temp1, ShifterOperand(temp3)); - __ b(&loop, NE); - - __ Bind(read_barrier_slow_path->GetExitLabel()); - } else { - // Non read barrier code. - // Compute the base source address in `temp1`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1); - // Compute the base destination address in `temp2`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2); - // Compute the end source address in `temp3`. - GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3); - // Iterate over the arrays and do a raw copy of the objects. We don't need to - // poison/unpoison. - Label loop; - __ Bind(&loop); - __ ldr(IP, Address(temp1, element_size, Address::PostIndex)); - __ str(IP, Address(temp2, element_size, Address::PostIndex)); - __ cmp(temp1, ShifterOperand(temp3)); - __ b(&loop, NE); - } - __ Bind(&done); - } - - // We only need one card marking on the destination array. - codegen_->MarkGCCard(temp1, temp2, dest, Register(kNoRegister), /* value_can_be_null */ false); - - __ Bind(intrinsic_slow_path->GetExitLabel()); -} - -static void CreateFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { - // If the graph is debuggable, all callee-saved floating-point registers are blocked by - // the code generator. Furthermore, the register allocator creates fixed live intervals - // for all caller-saved registers because we are doing a function call. As a result, if - // the input and output locations are unallocated, the register allocator runs out of - // registers and fails; however, a debuggable graph is not the common case. - if (invoke->GetBlock()->GetGraph()->IsDebuggable()) { - return; - } - - DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); - DCHECK_EQ(invoke->InputAt(0)->GetType(), Primitive::kPrimDouble); - DCHECK_EQ(invoke->GetType(), Primitive::kPrimDouble); - - LocationSummary* const locations = new (arena) LocationSummary(invoke, - LocationSummary::kCallOnMainOnly, - kIntrinsified); - const InvokeRuntimeCallingConvention calling_convention; - - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - // Native code uses the soft float ABI. - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); -} - -static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { - // If the graph is debuggable, all callee-saved floating-point registers are blocked by - // the code generator. Furthermore, the register allocator creates fixed live intervals - // for all caller-saved registers because we are doing a function call. As a result, if - // the input and output locations are unallocated, the register allocator runs out of - // registers and fails; however, a debuggable graph is not the common case. - if (invoke->GetBlock()->GetGraph()->IsDebuggable()) { - return; - } - - DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); - DCHECK_EQ(invoke->InputAt(0)->GetType(), Primitive::kPrimDouble); - DCHECK_EQ(invoke->InputAt(1)->GetType(), Primitive::kPrimDouble); - DCHECK_EQ(invoke->GetType(), Primitive::kPrimDouble); - - LocationSummary* const locations = new (arena) LocationSummary(invoke, - LocationSummary::kCallOnMainOnly, - kIntrinsified); - const InvokeRuntimeCallingConvention calling_convention; - - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - // Native code uses the soft float ABI. - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(3))); -} - -static void GenFPToFPCall(HInvoke* invoke, - ArmAssembler* assembler, - CodeGeneratorARM* codegen, - QuickEntrypointEnum entry) { - LocationSummary* const locations = invoke->GetLocations(); - const InvokeRuntimeCallingConvention calling_convention; - - DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); - DCHECK(locations->WillCall() && locations->Intrinsified()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(0))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(1))); - - // Native code uses the soft float ABI. - __ vmovrrd(calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1), - FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); - codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); - __ vmovdrr(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()), - calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1)); -} - -static void GenFPFPToFPCall(HInvoke* invoke, - ArmAssembler* assembler, - CodeGeneratorARM* codegen, - QuickEntrypointEnum entry) { - LocationSummary* const locations = invoke->GetLocations(); - const InvokeRuntimeCallingConvention calling_convention; - - DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); - DCHECK(locations->WillCall() && locations->Intrinsified()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(0))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(1))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(2))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(3))); - - // Native code uses the soft float ABI. - __ vmovrrd(calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1), - FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); - __ vmovrrd(calling_convention.GetRegisterAt(2), - calling_convention.GetRegisterAt(3), - FromLowSToD(locations->InAt(1).AsFpuRegisterPairLow<SRegister>())); - codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); - __ vmovdrr(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()), - calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1)); -} - -void IntrinsicLocationsBuilderARM::VisitMathCos(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathCos(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCos); -} - -void IntrinsicLocationsBuilderARM::VisitMathSin(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathSin(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickSin); -} - -void IntrinsicLocationsBuilderARM::VisitMathAcos(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAcos(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAcos); -} - -void IntrinsicLocationsBuilderARM::VisitMathAsin(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAsin(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAsin); -} - -void IntrinsicLocationsBuilderARM::VisitMathAtan(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAtan(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAtan); -} - -void IntrinsicLocationsBuilderARM::VisitMathCbrt(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathCbrt(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCbrt); -} - -void IntrinsicLocationsBuilderARM::VisitMathCosh(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathCosh(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCosh); -} - -void IntrinsicLocationsBuilderARM::VisitMathExp(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathExp(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickExp); -} - -void IntrinsicLocationsBuilderARM::VisitMathExpm1(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathExpm1(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickExpm1); -} - -void IntrinsicLocationsBuilderARM::VisitMathLog(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathLog(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickLog); -} - -void IntrinsicLocationsBuilderARM::VisitMathLog10(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathLog10(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickLog10); -} - -void IntrinsicLocationsBuilderARM::VisitMathSinh(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathSinh(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickSinh); -} - -void IntrinsicLocationsBuilderARM::VisitMathTan(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathTan(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickTan); -} - -void IntrinsicLocationsBuilderARM::VisitMathTanh(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathTanh(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickTanh); -} - -void IntrinsicLocationsBuilderARM::VisitMathAtan2(HInvoke* invoke) { - CreateFPFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAtan2(HInvoke* invoke) { - GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAtan2); -} - -void IntrinsicLocationsBuilderARM::VisitMathHypot(HInvoke* invoke) { - CreateFPFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathHypot(HInvoke* invoke) { - GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickHypot); -} - -void IntrinsicLocationsBuilderARM::VisitMathNextAfter(HInvoke* invoke) { - CreateFPFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathNextAfter(HInvoke* invoke) { - GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickNextAfter); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerReverse(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerReverse(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register out = locations->Out().AsRegister<Register>(); - Register in = locations->InAt(0).AsRegister<Register>(); - - __ rbit(out, in); -} - -void IntrinsicLocationsBuilderARM::VisitLongReverse(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongReverse(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); - - __ rbit(out_reg_lo, in_reg_hi); - __ rbit(out_reg_hi, in_reg_lo); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerReverseBytes(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerReverseBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register out = locations->Out().AsRegister<Register>(); - Register in = locations->InAt(0).AsRegister<Register>(); - - __ rev(out, in); -} - -void IntrinsicLocationsBuilderARM::VisitLongReverseBytes(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongReverseBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); - - __ rev(out_reg_lo, in_reg_hi); - __ rev(out_reg_hi, in_reg_lo); -} - -void IntrinsicLocationsBuilderARM::VisitShortReverseBytes(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitShortReverseBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register out = locations->Out().AsRegister<Register>(); - Register in = locations->InAt(0).AsRegister<Register>(); - - __ revsh(out, in); -} - -static void GenBitCount(HInvoke* instr, Primitive::Type type, ArmAssembler* assembler) { - DCHECK(Primitive::IsIntOrLongType(type)) << type; - DCHECK_EQ(instr->GetType(), Primitive::kPrimInt); - DCHECK_EQ(Primitive::PrimitiveKind(instr->InputAt(0)->GetType()), type); - - bool is_long = type == Primitive::kPrimLong; - LocationSummary* locations = instr->GetLocations(); - Location in = locations->InAt(0); - Register src_0 = is_long ? in.AsRegisterPairLow<Register>() : in.AsRegister<Register>(); - Register src_1 = is_long ? in.AsRegisterPairHigh<Register>() : src_0; - SRegister tmp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - DRegister tmp_d = FromLowSToD(tmp_s); - Register out_r = locations->Out().AsRegister<Register>(); - - // Move data from core register(s) to temp D-reg for bit count calculation, then move back. - // According to Cortex A57 and A72 optimization guides, compared to transferring to full D-reg, - // transferring data from core reg to upper or lower half of vfp D-reg requires extra latency, - // That's why for integer bit count, we use 'vmov d0, r0, r0' instead of 'vmov d0[0], r0'. - __ vmovdrr(tmp_d, src_1, src_0); // Temp DReg |--src_1|--src_0| - __ vcntd(tmp_d, tmp_d); // Temp DReg |c|c|c|c|c|c|c|c| - __ vpaddld(tmp_d, tmp_d, 8, /* is_unsigned */ true); // Temp DReg |--c|--c|--c|--c| - __ vpaddld(tmp_d, tmp_d, 16, /* is_unsigned */ true); // Temp DReg |------c|------c| - if (is_long) { - __ vpaddld(tmp_d, tmp_d, 32, /* is_unsigned */ true); // Temp DReg |--------------c| - } - __ vmovrs(out_r, tmp_s); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerBitCount(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); - invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerBitCount(HInvoke* invoke) { - GenBitCount(invoke, Primitive::kPrimInt, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitLongBitCount(HInvoke* invoke) { - VisitIntegerBitCount(invoke); -} - -void IntrinsicCodeGeneratorARM::VisitLongBitCount(HInvoke* invoke) { - GenBitCount(invoke, Primitive::kPrimLong, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitStringGetCharsNoCheck(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetInAt(3, Location::RequiresRegister()); - locations->SetInAt(4, Location::RequiresRegister()); - - // Temporary registers to store lengths of strings and for calculations. - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitStringGetCharsNoCheck(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - // Check assumption that sizeof(Char) is 2 (used in scaling below). - const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); - DCHECK_EQ(char_size, 2u); - - // Location of data in char array buffer. - const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value(); - - // Location of char array data in string. - const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); - - // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin); - // Since getChars() calls getCharsNoCheck() - we use registers rather than constants. - Register srcObj = locations->InAt(0).AsRegister<Register>(); - Register srcBegin = locations->InAt(1).AsRegister<Register>(); - Register srcEnd = locations->InAt(2).AsRegister<Register>(); - Register dstObj = locations->InAt(3).AsRegister<Register>(); - Register dstBegin = locations->InAt(4).AsRegister<Register>(); - - Register num_chr = locations->GetTemp(0).AsRegister<Register>(); - Register src_ptr = locations->GetTemp(1).AsRegister<Register>(); - Register dst_ptr = locations->GetTemp(2).AsRegister<Register>(); - - Label done, compressed_string_loop; - Label* final_label = codegen_->GetFinalLabel(invoke, &done); - // dst to be copied. - __ add(dst_ptr, dstObj, ShifterOperand(data_offset)); - __ add(dst_ptr, dst_ptr, ShifterOperand(dstBegin, LSL, 1)); - - __ subs(num_chr, srcEnd, ShifterOperand(srcBegin)); - // Early out for valid zero-length retrievals. - __ b(final_label, EQ); - - // src range to copy. - __ add(src_ptr, srcObj, ShifterOperand(value_offset)); - Label compressed_string_preloop; - if (mirror::kUseStringCompression) { - // Location of count in string. - const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); - // String's length. - __ ldr(IP, Address(srcObj, count_offset)); - __ tst(IP, ShifterOperand(1)); - __ b(&compressed_string_preloop, EQ); - } - __ add(src_ptr, src_ptr, ShifterOperand(srcBegin, LSL, 1)); - - // Do the copy. - Label loop, remainder; - - // Save repairing the value of num_chr on the < 4 character path. - __ subs(IP, num_chr, ShifterOperand(4)); - __ b(&remainder, LT); - - // Keep the result of the earlier subs, we are going to fetch at least 4 characters. - __ mov(num_chr, ShifterOperand(IP)); - - // Main loop used for longer fetches loads and stores 4x16-bit characters at a time. - // (LDRD/STRD fault on unaligned addresses and it's not worth inlining extra code - // to rectify these everywhere this intrinsic applies.) - __ Bind(&loop); - __ ldr(IP, Address(src_ptr, char_size * 2)); - __ subs(num_chr, num_chr, ShifterOperand(4)); - __ str(IP, Address(dst_ptr, char_size * 2)); - __ ldr(IP, Address(src_ptr, char_size * 4, Address::PostIndex)); - __ str(IP, Address(dst_ptr, char_size * 4, Address::PostIndex)); - __ b(&loop, GE); - - __ adds(num_chr, num_chr, ShifterOperand(4)); - __ b(final_label, EQ); - - // Main loop for < 4 character case and remainder handling. Loads and stores one - // 16-bit Java character at a time. - __ Bind(&remainder); - __ ldrh(IP, Address(src_ptr, char_size, Address::PostIndex)); - __ subs(num_chr, num_chr, ShifterOperand(1)); - __ strh(IP, Address(dst_ptr, char_size, Address::PostIndex)); - __ b(&remainder, GT); - - if (mirror::kUseStringCompression) { - __ b(final_label); - - const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte); - DCHECK_EQ(c_char_size, 1u); - // Copy loop for compressed src, copying 1 character (8-bit) to (16-bit) at a time. - __ Bind(&compressed_string_preloop); - __ add(src_ptr, src_ptr, ShifterOperand(srcBegin)); - __ Bind(&compressed_string_loop); - __ ldrb(IP, Address(src_ptr, c_char_size, Address::PostIndex)); - __ strh(IP, Address(dst_ptr, char_size, Address::PostIndex)); - __ subs(num_chr, num_chr, ShifterOperand(1)); - __ b(&compressed_string_loop, GT); - } - - if (done.IsLinked()) { - __ Bind(&done); - } -} - -void IntrinsicLocationsBuilderARM::VisitFloatIsInfinite(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitFloatIsInfinite(HInvoke* invoke) { - ArmAssembler* const assembler = GetAssembler(); - LocationSummary* const locations = invoke->GetLocations(); - const Register out = locations->Out().AsRegister<Register>(); - // Shifting left by 1 bit makes the value encodable as an immediate operand; - // we don't care about the sign bit anyway. - constexpr uint32_t infinity = kPositiveInfinityFloat << 1U; - - __ vmovrs(out, locations->InAt(0).AsFpuRegister<SRegister>()); - // We don't care about the sign bit, so shift left. - __ Lsl(out, out, 1); - __ eor(out, out, ShifterOperand(infinity)); - codegen_->GenerateConditionWithZero(kCondEQ, out, out); -} - -void IntrinsicLocationsBuilderARM::VisitDoubleIsInfinite(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitDoubleIsInfinite(HInvoke* invoke) { - ArmAssembler* const assembler = GetAssembler(); - LocationSummary* const locations = invoke->GetLocations(); - const Register out = locations->Out().AsRegister<Register>(); - // The highest 32 bits of double precision positive infinity separated into - // two constants encodable as immediate operands. - constexpr uint32_t infinity_high = 0x7f000000U; - constexpr uint32_t infinity_high2 = 0x00f00000U; - - static_assert((infinity_high | infinity_high2) == static_cast<uint32_t>(kPositiveInfinityDouble >> 32U), - "The constants do not add up to the high 32 bits of double precision positive infinity."); - __ vmovrrd(IP, out, FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); - __ eor(out, out, ShifterOperand(infinity_high)); - __ eor(out, out, ShifterOperand(infinity_high2)); - // We don't care about the sign bit, so shift left. - __ orr(out, IP, ShifterOperand(out, LSL, 1)); - codegen_->GenerateConditionWithZero(kCondEQ, out, out); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerValueOf(HInvoke* invoke) { - InvokeRuntimeCallingConvention calling_convention; - IntrinsicVisitor::ComputeIntegerValueOfLocations( - invoke, - codegen_, - Location::RegisterLocation(R0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0))); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerValueOf(HInvoke* invoke) { - IntrinsicVisitor::IntegerValueOfInfo info = IntrinsicVisitor::ComputeIntegerValueOfInfo(); - LocationSummary* locations = invoke->GetLocations(); - ArmAssembler* const assembler = GetAssembler(); - - Register out = locations->Out().AsRegister<Register>(); - InvokeRuntimeCallingConvention calling_convention; - Register argument = calling_convention.GetRegisterAt(0); - if (invoke->InputAt(0)->IsConstant()) { - int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); - if (value >= info.low && value <= info.high) { - // Just embed the j.l.Integer in the code. - ScopedObjectAccess soa(Thread::Current()); - mirror::Object* boxed = info.cache->Get(value + (-info.low)); - DCHECK(boxed != nullptr && Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(boxed)); - uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(boxed)); - __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); - } else { - // Allocate and initialize a new j.l.Integer. - // TODO: If we JIT, we could allocate the j.l.Integer now, and store it in the - // JIT object table. - uint32_t address = - dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer)); - __ LoadLiteral(argument, codegen_->DeduplicateBootImageAddressLiteral(address)); - codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); - CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); - __ LoadImmediate(IP, value); - __ StoreToOffset(kStoreWord, IP, out, info.value_offset); - // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation - // one. - codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore); - } - } else { - Register in = locations->InAt(0).AsRegister<Register>(); - // Check bounds of our cache. - __ AddConstant(out, in, -info.low); - __ CmpConstant(out, info.high - info.low + 1); - Label allocate, done; - __ b(&allocate, HS); - // If the value is within the bounds, load the j.l.Integer directly from the array. - uint32_t data_offset = mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); - uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.cache)); - __ LoadLiteral(IP, codegen_->DeduplicateBootImageAddressLiteral(data_offset + address)); - codegen_->LoadFromShiftedRegOffset(Primitive::kPrimNot, locations->Out(), IP, out); - __ MaybeUnpoisonHeapReference(out); - __ b(&done); - __ Bind(&allocate); - // Otherwise allocate and initialize a new j.l.Integer. - address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer)); - __ LoadLiteral(argument, codegen_->DeduplicateBootImageAddressLiteral(address)); - codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); - CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); - __ StoreToOffset(kStoreWord, in, out, info.value_offset); - // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation - // one. - codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore); - __ Bind(&done); - } -} - -void IntrinsicLocationsBuilderARM::VisitThreadInterrupted(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetOut(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitThreadInterrupted(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - Register out = invoke->GetLocations()->Out().AsRegister<Register>(); - int32_t offset = Thread::InterruptedOffset<kArmPointerSize>().Int32Value(); - __ LoadFromOffset(kLoadWord, out, TR, offset); - Label done; - Label* const final_label = codegen_->GetFinalLabel(invoke, &done); - __ CompareAndBranchIfZero(out, final_label); - __ dmb(ISH); - __ LoadImmediate(IP, 0); - __ StoreToOffset(kStoreWord, IP, TR, offset); - __ dmb(ISH); - if (done.IsLinked()) { - __ Bind(&done); - } -} - -UNIMPLEMENTED_INTRINSIC(ARM, MathMinDoubleDouble) -UNIMPLEMENTED_INTRINSIC(ARM, MathMinFloatFloat) -UNIMPLEMENTED_INTRINSIC(ARM, MathMaxDoubleDouble) -UNIMPLEMENTED_INTRINSIC(ARM, MathMaxFloatFloat) -UNIMPLEMENTED_INTRINSIC(ARM, MathMinLongLong) -UNIMPLEMENTED_INTRINSIC(ARM, MathMaxLongLong) -UNIMPLEMENTED_INTRINSIC(ARM, MathCeil) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, MathFloor) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, MathRint) -UNIMPLEMENTED_INTRINSIC(ARM, MathRoundDouble) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, MathRoundFloat) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeCASLong) // High register pressure. -UNIMPLEMENTED_INTRINSIC(ARM, SystemArrayCopyChar) -UNIMPLEMENTED_INTRINSIC(ARM, ReferenceGetReferent) -UNIMPLEMENTED_INTRINSIC(ARM, IntegerHighestOneBit) -UNIMPLEMENTED_INTRINSIC(ARM, LongHighestOneBit) -UNIMPLEMENTED_INTRINSIC(ARM, IntegerLowestOneBit) -UNIMPLEMENTED_INTRINSIC(ARM, LongLowestOneBit) - -UNIMPLEMENTED_INTRINSIC(ARM, StringStringIndexOf); -UNIMPLEMENTED_INTRINSIC(ARM, StringStringIndexOfAfter); -UNIMPLEMENTED_INTRINSIC(ARM, StringBufferAppend); -UNIMPLEMENTED_INTRINSIC(ARM, StringBufferLength); -UNIMPLEMENTED_INTRINSIC(ARM, StringBufferToString); -UNIMPLEMENTED_INTRINSIC(ARM, StringBuilderAppend); -UNIMPLEMENTED_INTRINSIC(ARM, StringBuilderLength); -UNIMPLEMENTED_INTRINSIC(ARM, StringBuilderToString); - -// 1.8. -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndAddInt) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndAddLong) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndSetInt) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndSetLong) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndSetObject) - -UNREACHABLE_INTRINSICS(ARM) - -#undef __ - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/intrinsics_arm.h b/compiler/optimizing/intrinsics_arm.h deleted file mode 100644 index 2840863632..0000000000 --- a/compiler/optimizing/intrinsics_arm.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_H_ -#define ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_H_ - -#include "intrinsics.h" - -namespace art { - -class ArenaAllocator; -class ArmInstructionSetFeatures; -class HInvokeStaticOrDirect; -class HInvokeVirtual; - -namespace arm { - -class ArmAssembler; -class CodeGeneratorARM; - -class IntrinsicLocationsBuilderARM FINAL : public IntrinsicVisitor { - public: - explicit IntrinsicLocationsBuilderARM(CodeGeneratorARM* codegen); - - // Define visitor methods. - -#define OPTIMIZING_INTRINSICS(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \ - void Visit ## Name(HInvoke* invoke) OVERRIDE; -#include "intrinsics_list.h" -INTRINSICS_LIST(OPTIMIZING_INTRINSICS) -#undef INTRINSICS_LIST -#undef OPTIMIZING_INTRINSICS - - // Check whether an invoke is an intrinsic, and if so, create a location summary. Returns whether - // a corresponding LocationSummary with the intrinsified_ flag set was generated and attached to - // the invoke. - bool TryDispatch(HInvoke* invoke); - - private: - ArenaAllocator* arena_; - CodeGenerator* codegen_; - ArmAssembler* assembler_; - - const ArmInstructionSetFeatures& features_; - - DISALLOW_COPY_AND_ASSIGN(IntrinsicLocationsBuilderARM); -}; - -class IntrinsicCodeGeneratorARM FINAL : public IntrinsicVisitor { - public: - explicit IntrinsicCodeGeneratorARM(CodeGeneratorARM* codegen) : codegen_(codegen) {} - - // Define visitor methods. - -#define OPTIMIZING_INTRINSICS(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \ - void Visit ## Name(HInvoke* invoke) OVERRIDE; -#include "intrinsics_list.h" -INTRINSICS_LIST(OPTIMIZING_INTRINSICS) -#undef INTRINSICS_LIST -#undef OPTIMIZING_INTRINSICS - - private: - ArmAssembler* GetAssembler(); - - ArenaAllocator* GetAllocator(); - - CodeGeneratorARM* codegen_; - - DISALLOW_COPY_AND_ASSIGN(IntrinsicCodeGeneratorARM); -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_H_ diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index 990a773a95..5691dd0d4a 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -124,12 +124,12 @@ class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 { // are no pools emitted. vixl::EmissionCheckScope guard(codegen->GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), - LocationFrom(kArtMethodRegister)); + codegen->GenerateStaticOrDirectCall( + invoke_->AsInvokeStaticOrDirect(), LocationFrom(kArtMethodRegister), this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister)); + codegen->GenerateVirtualCall( + invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister), this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); } // Copy the result back to the expected output. @@ -205,7 +205,7 @@ class ReadBarrierSystemArrayCopySlowPathARM64 : public SlowPathCodeARM64 { // TODO: Load the entrypoint once before the loop, instead of // loading it at every iteration. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg()); // This runtime call does not require a stack map. codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); codegen->GetAssembler()->MaybePoisonHeapReference(tmp_reg); @@ -2738,7 +2738,7 @@ void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) { // TODO: Also convert this intrinsic to the IsGcMarking strategy? // SystemArrayCopy implementation for Baker read barriers (see - // also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier): + // also CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier): // // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index 0e04b9a950..8b4044d69b 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -97,11 +97,10 @@ class IntrinsicSlowPathARMVIXL : public SlowPathCodeARMVIXL { Location method_loc = MoveArguments(codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc); + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc, this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc); + codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc, this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); @@ -227,7 +226,7 @@ class ReadBarrierSystemArrayCopySlowPathARMVIXL : public SlowPathCodeARMVIXL { // TODO: Load the entrypoint once before the loop, instead of // loading it at every iteration. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp.GetCode()); + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp.GetCode()); // This runtime call does not require a stack map. arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); assembler->MaybePoisonHeapReference(tmp); @@ -1059,7 +1058,7 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { // We need a temporary register for the read barrier marking slow - // path in InstructionCodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier. + // path in CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier. locations->AddTemp(Location::RequiresRegister()); } } @@ -2378,7 +2377,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { // TODO: Also convert this intrinsic to the IsGcMarking strategy? // SystemArrayCopy implementation for Baker read barriers (see - // also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier): + // also CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier): // // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index ea3e9e5ec9..4cea6dfdfb 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -112,12 +112,12 @@ class IntrinsicSlowPathMIPS : public SlowPathCodeMIPS { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), - Location::RegisterLocation(A0)); + codegen->GenerateStaticOrDirectCall( + invoke_->AsInvokeStaticOrDirect(), Location::RegisterLocation(A0), this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0)); + codegen->GenerateVirtualCall( + invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0), this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index 2ecb1a3b02..d785567e0f 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -101,12 +101,12 @@ class IntrinsicSlowPathMIPS64 : public SlowPathCodeMIPS64 { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), - Location::RegisterLocation(A0)); + codegen->GenerateStaticOrDirectCall( + invoke_->AsInvokeStaticOrDirect(), Location::RegisterLocation(A0), this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0)); + codegen->GenerateVirtualCall( + invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0), this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_utils.h b/compiler/optimizing/intrinsics_utils.h index c1f9ae6425..8c69d9b643 100644 --- a/compiler/optimizing/intrinsics_utils.h +++ b/compiler/optimizing/intrinsics_utils.h @@ -56,11 +56,10 @@ class IntrinsicSlowPath : public SlowPathCode { Location method_loc = MoveArguments(codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc); + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc, this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc); + codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc, this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index a9da15d2ce..a18b0cc400 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -143,8 +143,7 @@ class ReadBarrierSystemArrayCopySlowPathX86 : public SlowPathCode { // explanations.) DCHECK_NE(temp2, ESP); DCHECK(0 <= temp2 && temp2 < kNumberOfCpuRegisters) << temp2; - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(temp2); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(temp2); // This runtime call does not require a stack map. x86_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ MaybePoisonHeapReference(temp2); @@ -796,7 +795,6 @@ static void InvokeOutOfLineIntrinsic(CodeGeneratorX86* codegen, HInvoke* invoke) DCHECK(invoke->IsInvokeStaticOrDirect()); codegen->GenerateStaticOrDirectCall(invoke->AsInvokeStaticOrDirect(), Location::RegisterLocation(EAX)); - codegen->RecordPcInfo(invoke, invoke->GetDexPc()); // Copy the result back to the expected output. Location out = invoke->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index 8100645e54..5abdb1d1bd 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -105,8 +105,7 @@ class ReadBarrierSystemArrayCopySlowPathX86_64 : public SlowPathCode { // No need to save live registers; it's taken care of by the // entrypoint. Also, there is no need to update the stack mask, // as this runtime call will not trigger a garbage collection. - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(TMP); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(TMP); // This runtime call does not require a stack map. x86_64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ MaybePoisonHeapReference(CpuRegister(TMP)); @@ -567,7 +566,6 @@ static void InvokeOutOfLineIntrinsic(CodeGeneratorX86_64* codegen, HInvoke* invo DCHECK(invoke->IsInvokeStaticOrDirect()); codegen->GenerateStaticOrDirectCall( invoke->AsInvokeStaticOrDirect(), Location::RegisterLocation(RDI)); - codegen->RecordPcInfo(invoke, invoke->GetDexPc()); // Copy the result back to the expected output. Location out = invoke->GetLocations()->Out(); diff --git a/compiler/optimizing/load_store_analysis.cc b/compiler/optimizing/load_store_analysis.cc index f2ee345c8c..5a8ac59195 100644 --- a/compiler/optimizing/load_store_analysis.cc +++ b/compiler/optimizing/load_store_analysis.cc @@ -22,6 +22,117 @@ namespace art { // The number of heap locations for most of the methods stays below this threshold. constexpr size_t kMaxNumberOfHeapLocations = 32; +// Check if array indices array[idx1 +/- CONST] and array[idx2] MAY alias. +static bool BinaryOpAndIndexMayAlias(const HBinaryOperation* idx1, const HInstruction* idx2) { + DCHECK(idx1 != nullptr); + DCHECK(idx2 != nullptr); + + if (!idx1->IsAdd() && !idx1->IsSub()) { + // We currently only support Add and Sub operations. + return true; + } + + HConstant* cst = idx1->GetConstantRight(); + if (cst == nullptr || cst->IsArithmeticZero()) { + return true; + } + + if (idx1->GetLeastConstantLeft() == idx2) { + // for example, array[idx1 + 1] and array[idx1] + return false; + } + + return true; +} + +// Check if Add and Sub MAY alias when used as indices in arrays. +static bool BinaryOpsMayAlias(const HBinaryOperation* idx1, const HBinaryOperation* idx2) { + DCHECK(idx1!= nullptr); + DCHECK(idx2 != nullptr); + + HConstant* idx1_cst = idx1->GetConstantRight(); + HInstruction* idx1_other = idx1->GetLeastConstantLeft(); + HConstant* idx2_cst = idx2->GetConstantRight(); + HInstruction* idx2_other = idx2->GetLeastConstantLeft(); + + if (idx1_cst == nullptr || idx1_other == nullptr || + idx2_cst == nullptr || idx2_other == nullptr) { + // We only analyze patterns like [i +/- CONST]. + return true; + } + + if (idx1_other != idx2_other) { + // For example, [j+1] and [k+1] MAY alias. + return true; + } + + if ((idx1->IsAdd() && idx2->IsAdd()) || + (idx1->IsSub() && idx2->IsSub())) { + return idx1_cst->AsIntConstant()->GetValue() == idx2_cst->AsIntConstant()->GetValue(); + } + + if ((idx1->IsAdd() && idx2->IsSub()) || + (idx1->IsSub() && idx2->IsAdd())) { + // [i + CONST1] and [i - CONST2] MAY alias iff CONST1 == -CONST2. + // By checking CONST1 == -CONST2, following cases are handled: + // - Zero constants case [i+0] and [i-0] is handled. + // - Overflow cases are handled, for example: + // [i+0x80000000] and [i-0x80000000]; + // [i+0x10] and [i-0xFFFFFFF0]. + // - Other cases [i+CONST1] and [i-CONST2] without any overflow is handled. + return idx1_cst->AsIntConstant()->GetValue() == -(idx2_cst->AsIntConstant()->GetValue()); + } + + // All other cases, MAY alias. + return true; +} + +// The following array index cases are handled: +// [i] and [i] +// [CONST1] and [CONST2] +// [i] and [i+CONST] +// [i] and [i-CONST] +// [i+CONST1] and [i+CONST2] +// [i-CONST1] and [i-CONST2] +// [i+CONST1] and [i-CONST2] +// [i-CONST1] and [i+CONST2] +// For other complicated cases, we rely on other passes like GVN and simpilfier +// to optimize these cases before this pass. +// For example: [i+j+k+10] and [i+k+10+j] shall be optimized to [i7+10] and [i7+10]. +bool HeapLocationCollector::CanArrayIndicesAlias(const HInstruction* idx1, + const HInstruction* idx2) const { + DCHECK(idx1 != nullptr); + DCHECK(idx2 != nullptr); + + if (idx1 == idx2) { + // [i] and [i] + return true; + } + if (idx1->IsIntConstant() && idx2->IsIntConstant()) { + // [CONST1] and [CONST2] + return idx1->AsIntConstant()->GetValue() == idx2->AsIntConstant()->GetValue(); + } + + if (idx1->IsBinaryOperation() && !BinaryOpAndIndexMayAlias(idx1->AsBinaryOperation(), idx2)) { + // [i] and [i+/-CONST] + return false; + } + if (idx2->IsBinaryOperation() && !BinaryOpAndIndexMayAlias(idx2->AsBinaryOperation(), idx1)) { + // [i+/-CONST] and [i] + return false; + } + + if (idx1->IsBinaryOperation() && idx2->IsBinaryOperation()) { + // [i+/-CONST1] and [i+/-CONST2] + if (!BinaryOpsMayAlias(idx1->AsBinaryOperation(), idx2->AsBinaryOperation())) { + return false; + } + } + + // By default, MAY alias. + return true; +} + void LoadStoreAnalysis::Run() { for (HBasicBlock* block : graph_->GetReversePostOrder()) { heap_location_collector_.VisitBasicBlock(block); diff --git a/compiler/optimizing/load_store_analysis.h b/compiler/optimizing/load_store_analysis.h index 4e940f30bf..a2c17944f2 100644 --- a/compiler/optimizing/load_store_analysis.h +++ b/compiler/optimizing/load_store_analysis.h @@ -214,6 +214,17 @@ class HeapLocationCollector : public HGraphVisitor { return nullptr; } + size_t GetArrayAccessHeapLocation(HInstruction* array, HInstruction* index) const { + DCHECK(array != nullptr); + DCHECK(index != nullptr); + HInstruction* original_ref = HuntForOriginalReference(array); + ReferenceInfo* ref_info = FindReferenceInfoOf(original_ref); + return FindHeapLocationIndex(ref_info, + HeapLocation::kInvalidFieldOffset, + index, + HeapLocation::kDeclaringClassDefIndexForArrays); + } + bool HasHeapStores() const { return has_heap_stores_; } @@ -300,6 +311,8 @@ class HeapLocationCollector : public HGraphVisitor { return true; } + bool CanArrayIndicesAlias(const HInstruction* i1, const HInstruction* i2) const; + // `index1` and `index2` are indices in the array of collected heap locations. // Returns the position in the bit vector that tracks whether the two heap // locations may alias. @@ -336,12 +349,7 @@ class HeapLocationCollector : public HGraphVisitor { if (loc1->IsArrayElement() && loc2->IsArrayElement()) { HInstruction* array_index1 = loc1->GetIndex(); HInstruction* array_index2 = loc2->GetIndex(); - DCHECK(array_index1 != nullptr); - DCHECK(array_index2 != nullptr); - if (array_index1->IsIntConstant() && - array_index2->IsIntConstant() && - array_index1->AsIntConstant()->GetValue() != array_index2->AsIntConstant()->GetValue()) { - // Different constant indices do not alias. + if (!CanArrayIndicesAlias(array_index1, array_index2)) { return false; } ReferenceInfo* ref_info = loc1->GetReferenceInfo(); @@ -458,6 +466,11 @@ class HeapLocationCollector : public HGraphVisitor { CreateReferenceInfoForReferenceType(new_instance); } + void VisitNewArray(HNewArray* new_array) OVERRIDE { + // Any references appearing in the ref_info_array_ so far cannot alias with new_array. + CreateReferenceInfoForReferenceType(new_array); + } + void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* instruction) OVERRIDE { CreateReferenceInfoForReferenceType(instruction); } @@ -470,6 +483,22 @@ class HeapLocationCollector : public HGraphVisitor { CreateReferenceInfoForReferenceType(instruction); } + void VisitInvokeUnresolved(HInvokeUnresolved* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + + void VisitInvokePolymorphic(HInvokePolymorphic* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + + void VisitLoadString(HLoadString* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + + void VisitPhi(HPhi* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + void VisitParameterValue(HParameterValue* instruction) OVERRIDE { CreateReferenceInfoForReferenceType(instruction); } diff --git a/compiler/optimizing/load_store_analysis_test.cc b/compiler/optimizing/load_store_analysis_test.cc index 24187777f6..81344b52f6 100644 --- a/compiler/optimizing/load_store_analysis_test.cc +++ b/compiler/optimizing/load_store_analysis_test.cc @@ -184,4 +184,198 @@ TEST_F(LoadStoreAnalysisTest, FieldHeapLocations) { ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); } +TEST_F(LoadStoreAnalysisTest, ArrayIndexAliasingTest) { + HBasicBlock* entry = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(entry); + graph_->SetEntryBlock(entry); + graph_->BuildDominatorTree(); + + HInstruction* array = new (&allocator_) HParameterValue( + graph_->GetDexFile(), dex::TypeIndex(0), 0, Primitive::kPrimNot); + HInstruction* index = new (&allocator_) HParameterValue( + graph_->GetDexFile(), dex::TypeIndex(1), 1, Primitive::kPrimInt); + HInstruction* c0 = graph_->GetIntConstant(0); + HInstruction* c1 = graph_->GetIntConstant(1); + HInstruction* c_neg1 = graph_->GetIntConstant(-1); + HInstruction* add0 = new (&allocator_) HAdd(Primitive::kPrimInt, index, c0); + HInstruction* add1 = new (&allocator_) HAdd(Primitive::kPrimInt, index, c1); + HInstruction* sub0 = new (&allocator_) HSub(Primitive::kPrimInt, index, c0); + HInstruction* sub1 = new (&allocator_) HSub(Primitive::kPrimInt, index, c1); + HInstruction* sub_neg1 = new (&allocator_) HSub(Primitive::kPrimInt, index, c_neg1); + HInstruction* rev_sub1 = new (&allocator_) HSub(Primitive::kPrimInt, c1, index); + HInstruction* arr_set1 = new (&allocator_) HArraySet(array, c0, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set2 = new (&allocator_) HArraySet(array, c1, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set3 = new (&allocator_) HArraySet(array, add0, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set4 = new (&allocator_) HArraySet(array, add1, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set5 = new (&allocator_) HArraySet(array, sub0, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set6 = new (&allocator_) HArraySet(array, sub1, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set7 = new (&allocator_) HArraySet(array, rev_sub1, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set8 = new (&allocator_) HArraySet(array, sub_neg1, c0, Primitive::kPrimInt, 0); + + entry->AddInstruction(array); + entry->AddInstruction(index); + entry->AddInstruction(add0); + entry->AddInstruction(add1); + entry->AddInstruction(sub0); + entry->AddInstruction(sub1); + entry->AddInstruction(sub_neg1); + entry->AddInstruction(rev_sub1); + + entry->AddInstruction(arr_set1); // array[0] = c0 + entry->AddInstruction(arr_set2); // array[1] = c0 + entry->AddInstruction(arr_set3); // array[i+0] = c0 + entry->AddInstruction(arr_set4); // array[i+1] = c0 + entry->AddInstruction(arr_set5); // array[i-0] = c0 + entry->AddInstruction(arr_set6); // array[i-1] = c0 + entry->AddInstruction(arr_set7); // array[1-i] = c0 + entry->AddInstruction(arr_set8); // array[i-(-1)] = c0 + + LoadStoreAnalysis lsa(graph_); + lsa.Run(); + const HeapLocationCollector& heap_location_collector = lsa.GetHeapLocationCollector(); + + // LSA/HeapLocationCollector should see those ArrayGet instructions. + ASSERT_EQ(heap_location_collector.GetNumberOfHeapLocations(), 8U); + ASSERT_TRUE(heap_location_collector.HasHeapStores()); + + // Test queries on HeapLocationCollector's aliasing matrix after load store analysis. + size_t loc1 = HeapLocationCollector::kHeapLocationNotFound; + size_t loc2 = HeapLocationCollector::kHeapLocationNotFound; + + // Test alias: array[0] and array[1] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, c0); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, c1); + ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); + + // Test alias: array[i+0] and array[i-0] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add0); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub0); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + + // Test alias: array[i+1] and array[i-1] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add1); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub1); + ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); + + // Test alias: array[i+1] and array[1-i] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add1); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, rev_sub1); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + + // Test alias: array[i+1] and array[i-(-1)] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add1); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_neg1); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); +} + +TEST_F(LoadStoreAnalysisTest, ArrayIndexCalculationOverflowTest) { + HBasicBlock* entry = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(entry); + graph_->SetEntryBlock(entry); + graph_->BuildDominatorTree(); + + HInstruction* array = new (&allocator_) HParameterValue( + graph_->GetDexFile(), dex::TypeIndex(0), 0, Primitive::kPrimNot); + HInstruction* index = new (&allocator_) HParameterValue( + graph_->GetDexFile(), dex::TypeIndex(1), 1, Primitive::kPrimInt); + + HInstruction* c0 = graph_->GetIntConstant(0); + HInstruction* c_0x80000000 = graph_->GetIntConstant(0x80000000); + HInstruction* c_0x10 = graph_->GetIntConstant(0x10); + HInstruction* c_0xFFFFFFF0 = graph_->GetIntConstant(0xFFFFFFF0); + HInstruction* c_0x7FFFFFFF = graph_->GetIntConstant(0x7FFFFFFF); + HInstruction* c_0x80000001 = graph_->GetIntConstant(0x80000001); + + // `index+0x80000000` and `index-0x80000000` array indices MAY alias. + HInstruction* add_0x80000000 = new (&allocator_) HAdd(Primitive::kPrimInt, index, c_0x80000000); + HInstruction* sub_0x80000000 = new (&allocator_) HSub(Primitive::kPrimInt, index, c_0x80000000); + HInstruction* arr_set_1 = new (&allocator_) HArraySet( + array, add_0x80000000, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_2 = new (&allocator_) HArraySet( + array, sub_0x80000000, c0, Primitive::kPrimInt, 0); + + // `index+0x10` and `index-0xFFFFFFF0` array indices MAY alias. + HInstruction* add_0x10 = new (&allocator_) HAdd(Primitive::kPrimInt, index, c_0x10); + HInstruction* sub_0xFFFFFFF0 = new (&allocator_) HSub(Primitive::kPrimInt, index, c_0xFFFFFFF0); + HInstruction* arr_set_3 = new (&allocator_) HArraySet( + array, add_0x10, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_4 = new (&allocator_) HArraySet( + array, sub_0xFFFFFFF0, c0, Primitive::kPrimInt, 0); + + // `index+0x7FFFFFFF` and `index-0x80000001` array indices MAY alias. + HInstruction* add_0x7FFFFFFF = new (&allocator_) HAdd(Primitive::kPrimInt, index, c_0x7FFFFFFF); + HInstruction* sub_0x80000001 = new (&allocator_) HSub(Primitive::kPrimInt, index, c_0x80000001); + HInstruction* arr_set_5 = new (&allocator_) HArraySet( + array, add_0x7FFFFFFF, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_6 = new (&allocator_) HArraySet( + array, sub_0x80000001, c0, Primitive::kPrimInt, 0); + + // `index+0` and `index-0` array indices MAY alias. + HInstruction* add_0 = new (&allocator_) HAdd(Primitive::kPrimInt, index, c0); + HInstruction* sub_0 = new (&allocator_) HSub(Primitive::kPrimInt, index, c0); + HInstruction* arr_set_7 = new (&allocator_) HArraySet(array, add_0, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_8 = new (&allocator_) HArraySet(array, sub_0, c0, Primitive::kPrimInt, 0); + + entry->AddInstruction(array); + entry->AddInstruction(index); + entry->AddInstruction(add_0x80000000); + entry->AddInstruction(sub_0x80000000); + entry->AddInstruction(add_0x10); + entry->AddInstruction(sub_0xFFFFFFF0); + entry->AddInstruction(add_0x7FFFFFFF); + entry->AddInstruction(sub_0x80000001); + entry->AddInstruction(add_0); + entry->AddInstruction(sub_0); + entry->AddInstruction(arr_set_1); + entry->AddInstruction(arr_set_2); + entry->AddInstruction(arr_set_3); + entry->AddInstruction(arr_set_4); + entry->AddInstruction(arr_set_5); + entry->AddInstruction(arr_set_6); + entry->AddInstruction(arr_set_7); + entry->AddInstruction(arr_set_8); + + LoadStoreAnalysis lsa(graph_); + lsa.Run(); + const HeapLocationCollector& heap_location_collector = lsa.GetHeapLocationCollector(); + + // LSA/HeapLocationCollector should see those ArrayGet instructions. + ASSERT_EQ(heap_location_collector.GetNumberOfHeapLocations(), 8U); + ASSERT_TRUE(heap_location_collector.HasHeapStores()); + + // Test queries on HeapLocationCollector's aliasing matrix after load store analysis. + size_t loc1 = HeapLocationCollector::kHeapLocationNotFound; + size_t loc2 = HeapLocationCollector::kHeapLocationNotFound; + + // Test alias: array[i+0x80000000] and array[i-0x80000000] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add_0x80000000); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_0x80000000); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + + // Test alias: array[i+0x10] and array[i-0xFFFFFFF0] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add_0x10); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_0xFFFFFFF0); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + + // Test alias: array[i+0x7FFFFFFF] and array[i-0x80000001] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add_0x7FFFFFFF); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_0x80000001); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + + // Test alias: array[i+0] and array[i-0] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add_0); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_0); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + + // Should not alias: + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_0x80000000); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_0x80000001); + ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); + + // Should not alias: + loc1 = heap_location_collector.GetArrayAccessHeapLocation(array, add_0); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(array, sub_0x80000000); + ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); +} + } // namespace art diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index 4334ab10bd..422e58debb 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -31,6 +31,9 @@ namespace art { // Enables vectorization (SIMDization) in the loop optimizer. static constexpr bool kEnableVectorization = true; +// All current SIMD targets want 16-byte alignment. +static constexpr size_t kAlignedBase = 16; + // Remove the instruction from the graph. A bit more elaborate than the usual // instruction removal, since there may be a cycle in the use structure. static void RemoveFromCycle(HInstruction* instruction) { @@ -283,6 +286,9 @@ HLoopOptimization::HLoopOptimization(HGraph* graph, simplified_(false), vector_length_(0), vector_refs_(nullptr), + vector_peeling_candidate_(nullptr), + vector_runtime_test_a_(nullptr), + vector_runtime_test_b_(nullptr), vector_map_(nullptr) { } @@ -422,23 +428,6 @@ void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { // Optimization. // -bool HLoopOptimization::CanRemoveCycle() { - for (HInstruction* i : *iset_) { - // We can never remove instructions that have environment - // uses when we compile 'debuggable'. - if (i->HasEnvironmentUses() && graph_->IsDebuggable()) { - return false; - } - // A deoptimization should never have an environment input removed. - for (const HUseListNode<HEnvironment*>& use : i->GetEnvUses()) { - if (use.GetUser()->GetHolder()->IsDeoptimize()) { - return false; - } - } - } - return true; -} - void HLoopOptimization::SimplifyInduction(LoopNode* node) { HBasicBlock* header = node->loop_info->GetHeader(); HBasicBlock* preheader = node->loop_info->GetPreHeader(); @@ -521,6 +510,7 @@ void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { body = it.Current(); } } + CHECK(body != nullptr); // Ensure there is only a single exit point. if (header->GetSuccessors().size() != 2) { return; @@ -564,7 +554,7 @@ void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { if (kEnableVectorization) { iset_->clear(); // prepare phi induction if (TrySetSimpleLoopHeader(header) && - CanVectorize(node, body, trip_count) && + ShouldVectorize(node, body, trip_count) && TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ true)) { Vectorize(node, body, exit, trip_count); graph_->SetHasSIMD(true); // flag SIMD usage @@ -579,10 +569,11 @@ void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { // Intel Press, June, 2004 (http://www.aartbik.com/). // -bool HLoopOptimization::CanVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count) { +bool HLoopOptimization::ShouldVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count) { // Reset vector bookkeeping. vector_length_ = 0; vector_refs_->clear(); + vector_peeling_candidate_ = nullptr; vector_runtime_test_a_ = vector_runtime_test_b_= nullptr; @@ -599,12 +590,9 @@ bool HLoopOptimization::CanVectorize(LoopNode* node, HBasicBlock* block, int64_t } } - // Heuristics. Does vectorization seem profitable? - // TODO: refine - if (vector_length_ == 0) { - return false; // nothing found - } else if (0 < trip_count && trip_count < vector_length_) { - return false; // insufficient iterations + // Does vectorization seem profitable? + if (!IsVectorizationProfitable(trip_count)) { + return false; } // Data dependence analysis. Find each pair of references with same type, where @@ -632,18 +620,24 @@ bool HLoopOptimization::CanVectorize(LoopNode* node, HBasicBlock* block, int64_t // Conservatively assume a potential loop-carried data dependence otherwise, avoided by // generating an explicit a != b disambiguation runtime test on the two references. if (x != y) { - // For now, we reject after one test to avoid excessive overhead. - if (vector_runtime_test_a_ != nullptr) { - return false; + // To avoid excessive overhead, we only accept one a != b test. + if (vector_runtime_test_a_ == nullptr) { + // First test found. + vector_runtime_test_a_ = a; + vector_runtime_test_b_ = b; + } else if ((vector_runtime_test_a_ != a || vector_runtime_test_b_ != b) && + (vector_runtime_test_a_ != b || vector_runtime_test_b_ != a)) { + return false; // second test would be needed } - vector_runtime_test_a_ = a; - vector_runtime_test_b_ = b; } } } } } + // Consider dynamic loop peeling for alignment. + SetPeelingCandidate(trip_count); + // Success! return true; } @@ -656,28 +650,52 @@ void HLoopOptimization::Vectorize(LoopNode* node, HBasicBlock* header = node->loop_info->GetHeader(); HBasicBlock* preheader = node->loop_info->GetPreHeader(); - // A cleanup is needed for any unknown trip count or for a known trip count - // with remainder iterations after vectorization. - bool needs_cleanup = trip_count == 0 || (trip_count % vector_length_) != 0; + // Pick a loop unrolling factor for the vector loop. + uint32_t unroll = GetUnrollingFactor(block, trip_count); + uint32_t chunk = vector_length_ * unroll; + + // A cleanup loop is needed, at least, for any unknown trip count or + // for a known trip count with remainder iterations after vectorization. + bool needs_cleanup = trip_count == 0 || (trip_count % chunk) != 0; // Adjust vector bookkeeping. iset_->clear(); // prepare phi induction bool is_simple_loop_header = TrySetSimpleLoopHeader(header); // fills iset_ DCHECK(is_simple_loop_header); + vector_header_ = header; + vector_body_ = block; + + // Generate dynamic loop peeling trip count, if needed: + // ptc = <peeling-needed-for-candidate> + HInstruction* ptc = nullptr; + if (vector_peeling_candidate_ != nullptr) { + DCHECK_LT(vector_length_, trip_count) << "dynamic peeling currently requires known trip count"; + // + // TODO: Implement this. Compute address of first access memory location and + // compute peeling factor to obtain kAlignedBase alignment. + // + needs_cleanup = true; + } - // Generate preheader: + // Generate loop control: // stc = <trip-count>; - // vtc = stc - stc % VL; + // vtc = stc - (stc - ptc) % chunk; + // i = 0; HInstruction* stc = induction_range_.GenerateTripCount(node->loop_info, graph_, preheader); HInstruction* vtc = stc; if (needs_cleanup) { - DCHECK(IsPowerOfTwo(vector_length_)); + DCHECK(IsPowerOfTwo(chunk)); + HInstruction* diff = stc; + if (ptc != nullptr) { + diff = Insert(preheader, new (global_allocator_) HSub(induc_type, stc, ptc)); + } HInstruction* rem = Insert( preheader, new (global_allocator_) HAnd(induc_type, - stc, - graph_->GetIntConstant(vector_length_ - 1))); + diff, + graph_->GetIntConstant(chunk - 1))); vtc = Insert(preheader, new (global_allocator_) HSub(induc_type, stc, rem)); } + vector_index_ = graph_->GetIntConstant(0); // Generate runtime disambiguation test: // vtc = a != b ? vtc : 0; @@ -690,16 +708,31 @@ void HLoopOptimization::Vectorize(LoopNode* node, needs_cleanup = true; } - // Generate vector loop: - // for (i = 0; i < vtc; i += VL) + // Generate dynamic peeling loop for alignment, if needed: + // for ( ; i < ptc; i += 1) + // <loop-body> + if (ptc != nullptr) { + vector_mode_ = kSequential; + GenerateNewLoop(node, + block, + graph_->TransformLoopForVectorization(vector_header_, vector_body_, exit), + vector_index_, + ptc, + graph_->GetIntConstant(1), + /*unroll*/ 1); + } + + // Generate vector loop, possibly further unrolled: + // for ( ; i < vtc; i += chunk) // <vectorized-loop-body> vector_mode_ = kVector; GenerateNewLoop(node, block, - graph_->TransformLoopForVectorization(header, block, exit), - graph_->GetIntConstant(0), + graph_->TransformLoopForVectorization(vector_header_, vector_body_, exit), + vector_index_, vtc, - graph_->GetIntConstant(vector_length_)); + graph_->GetIntConstant(vector_length_), // increment per unroll + unroll); HLoopInformation* vloop = vector_header_->GetLoopInformation(); // Generate cleanup loop, if needed: @@ -710,9 +743,10 @@ void HLoopOptimization::Vectorize(LoopNode* node, GenerateNewLoop(node, block, graph_->TransformLoopForVectorization(vector_header_, vector_body_, exit), - vector_phi_, + vector_index_, stc, - graph_->GetIntConstant(1)); + graph_->GetIntConstant(1), + /*unroll*/ 1); } // Remove the original loop by disconnecting the body block @@ -721,8 +755,9 @@ void HLoopOptimization::Vectorize(LoopNode* node, while (!header->GetFirstInstruction()->IsGoto()) { header->RemoveInstruction(header->GetFirstInstruction()); } - // Update loop hierarchy: the old header now resides in the - // same outer loop as the old preheader. + // Update loop hierarchy: the old header now resides in the same outer loop + // as the old preheader. Note that we don't bother putting sequential + // loops back in the hierarchy at this point. header->SetLoopInformation(preheader->GetLoopInformation()); // outward node->loop_info = vloop; } @@ -732,44 +767,64 @@ void HLoopOptimization::GenerateNewLoop(LoopNode* node, HBasicBlock* new_preheader, HInstruction* lo, HInstruction* hi, - HInstruction* step) { + HInstruction* step, + uint32_t unroll) { + DCHECK(unroll == 1 || vector_mode_ == kVector); Primitive::Type induc_type = Primitive::kPrimInt; // Prepare new loop. - vector_map_->clear(); vector_preheader_ = new_preheader, vector_header_ = vector_preheader_->GetSingleSuccessor(); vector_body_ = vector_header_->GetSuccessors()[1]; - vector_phi_ = new (global_allocator_) HPhi(global_allocator_, - kNoRegNumber, - 0, - HPhi::ToPhiType(induc_type)); + HPhi* phi = new (global_allocator_) HPhi(global_allocator_, + kNoRegNumber, + 0, + HPhi::ToPhiType(induc_type)); // Generate header and prepare body. // for (i = lo; i < hi; i += step) // <loop-body> - HInstruction* cond = new (global_allocator_) HAboveOrEqual(vector_phi_, hi); - vector_header_->AddPhi(vector_phi_); + HInstruction* cond = new (global_allocator_) HAboveOrEqual(phi, hi); + vector_header_->AddPhi(phi); vector_header_->AddInstruction(cond); vector_header_->AddInstruction(new (global_allocator_) HIf(cond)); - for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { - bool vectorized_def = VectorizeDef(node, it.Current(), /*generate_code*/ true); - DCHECK(vectorized_def); - } - // Generate body from the instruction map, but in original program order. - HEnvironment* env = vector_header_->GetFirstInstruction()->GetEnvironment(); - for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { - auto i = vector_map_->find(it.Current()); - if (i != vector_map_->end() && !i->second->IsInBlock()) { - Insert(vector_body_, i->second); - // Deal with instructions that need an environment, such as the scalar intrinsics. - if (i->second->NeedsEnvironment()) { - i->second->CopyEnvironmentFromWithLoopPhiAdjustment(env, vector_header_); + vector_index_ = phi; + for (uint32_t u = 0; u < unroll; u++) { + // Clear map, leaving loop invariants setup during unrolling. + if (u == 0) { + vector_map_->clear(); + } else { + for (auto i = vector_map_->begin(); i != vector_map_->end(); ) { + if (i->second->IsVecReplicateScalar()) { + DCHECK(node->loop_info->IsDefinedOutOfTheLoop(i->first)); + ++i; + } else { + i = vector_map_->erase(i); + } + } + } + // Generate instruction map. + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + bool vectorized_def = VectorizeDef(node, it.Current(), /*generate_code*/ true); + DCHECK(vectorized_def); + } + // Generate body from the instruction map, but in original program order. + HEnvironment* env = vector_header_->GetFirstInstruction()->GetEnvironment(); + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + auto i = vector_map_->find(it.Current()); + if (i != vector_map_->end() && !i->second->IsInBlock()) { + Insert(vector_body_, i->second); + // Deal with instructions that need an environment, such as the scalar intrinsics. + if (i->second->NeedsEnvironment()) { + i->second->CopyEnvironmentFromWithLoopPhiAdjustment(env, vector_header_); + } } } + vector_index_ = new (global_allocator_) HAdd(induc_type, vector_index_, step); + Insert(vector_body_, vector_index_); } - // Finalize increment and phi. - HInstruction* inc = new (global_allocator_) HAdd(induc_type, vector_phi_, step); - vector_phi_->AddInput(lo); - vector_phi_->AddInput(Insert(vector_body_, inc)); + // Finalize phi for the loop index. + phi->AddInput(lo); + phi->AddInput(vector_index_); + vector_index_ = phi; } // TODO: accept reductions at left-hand-side, mixed-type store idioms, etc. @@ -790,11 +845,11 @@ bool HLoopOptimization::VectorizeDef(LoopNode* node, HInstruction* offset = nullptr; if (TrySetVectorType(type, &restrictions) && node->loop_info->IsDefinedOutOfTheLoop(base) && - induction_range_.IsUnitStride(instruction, index, &offset) && + induction_range_.IsUnitStride(instruction, index, graph_, &offset) && VectorizeUse(node, value, generate_code, type, restrictions)) { if (generate_code) { GenerateVecSub(index, offset); - GenerateVecMem(instruction, vector_map_->Get(index), vector_map_->Get(value), type); + GenerateVecMem(instruction, vector_map_->Get(index), vector_map_->Get(value), offset, type); } else { vector_refs_->insert(ArrayReference(base, offset, type, /*lhs*/ true)); } @@ -848,10 +903,10 @@ bool HLoopOptimization::VectorizeUse(LoopNode* node, HInstruction* offset = nullptr; if (type == instruction->GetType() && node->loop_info->IsDefinedOutOfTheLoop(base) && - induction_range_.IsUnitStride(instruction, index, &offset)) { + induction_range_.IsUnitStride(instruction, index, graph_, &offset)) { if (generate_code) { GenerateVecSub(index, offset); - GenerateVecMem(instruction, vector_map_->Get(index), nullptr, type); + GenerateVecMem(instruction, vector_map_->Get(index), nullptr, offset, type); } else { vector_refs_->insert(ArrayReference(base, offset, type, /*lhs*/ false)); } @@ -1042,6 +1097,23 @@ bool HLoopOptimization::TrySetVectorType(Primitive::Type type, uint64_t* restric switch (compiler_driver_->GetInstructionSet()) { case kArm: case kThumb2: + // Allow vectorization for all ARM devices, because Android assumes that + // ARM 32-bit always supports advanced SIMD. + switch (type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + *restrictions |= kNoDiv; + return TrySetVectorLength(8); + case Primitive::kPrimChar: + case Primitive::kPrimShort: + *restrictions |= kNoDiv | kNoStringCharAt; + return TrySetVectorLength(4); + case Primitive::kPrimInt: + *restrictions |= kNoDiv; + return TrySetVectorLength(2); + default: + break; + } return false; case kArm64: // Allow vectorization for all ARM devices, because Android assumes that @@ -1099,30 +1171,55 @@ bool HLoopOptimization::TrySetVectorType(Primitive::Type type, uint64_t* restric } return false; case kMips: - // TODO: implement MIPS SIMD. + if (features->AsMipsInstructionSetFeatures()->HasMsa()) { + switch (type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + *restrictions |= kNoDiv; + return TrySetVectorLength(16); + case Primitive::kPrimChar: + case Primitive::kPrimShort: + *restrictions |= kNoDiv | kNoStringCharAt; + return TrySetVectorLength(8); + case Primitive::kPrimInt: + *restrictions |= kNoDiv; + return TrySetVectorLength(4); + case Primitive::kPrimLong: + *restrictions |= kNoDiv; + return TrySetVectorLength(2); + case Primitive::kPrimFloat: + *restrictions |= kNoMinMax; // min/max(x, NaN) + return TrySetVectorLength(4); + case Primitive::kPrimDouble: + *restrictions |= kNoMinMax; // min/max(x, NaN) + return TrySetVectorLength(2); + default: + break; + } // switch type + } return false; case kMips64: if (features->AsMips64InstructionSetFeatures()->HasMsa()) { switch (type) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: - *restrictions |= kNoDiv | kNoMinMax; + *restrictions |= kNoDiv; return TrySetVectorLength(16); case Primitive::kPrimChar: case Primitive::kPrimShort: - *restrictions |= kNoDiv | kNoMinMax | kNoStringCharAt; + *restrictions |= kNoDiv | kNoStringCharAt; return TrySetVectorLength(8); case Primitive::kPrimInt: - *restrictions |= kNoDiv | kNoMinMax; + *restrictions |= kNoDiv; return TrySetVectorLength(4); case Primitive::kPrimLong: - *restrictions |= kNoDiv | kNoMinMax; + *restrictions |= kNoDiv; return TrySetVectorLength(2); case Primitive::kPrimFloat: - *restrictions |= kNoMinMax; + *restrictions |= kNoMinMax; // min/max(x, NaN) return TrySetVectorLength(4); case Primitive::kPrimDouble: - *restrictions |= kNoMinMax; + *restrictions |= kNoMinMax; // min/max(x, NaN) return TrySetVectorLength(2); default: break; @@ -1163,8 +1260,9 @@ void HLoopOptimization::GenerateVecInv(HInstruction* org, Primitive::Type type) void HLoopOptimization::GenerateVecSub(HInstruction* org, HInstruction* offset) { if (vector_map_->find(org) == vector_map_->end()) { - HInstruction* subscript = vector_phi_; - if (offset != nullptr) { + HInstruction* subscript = vector_index_; + int64_t value = 0; + if (!IsInt64AndGet(offset, &value) || value != 0) { subscript = new (global_allocator_) HAdd(Primitive::kPrimInt, subscript, offset); if (org->IsPhi()) { Insert(vector_body_, subscript); // lacks layout placeholder @@ -1177,17 +1275,27 @@ void HLoopOptimization::GenerateVecSub(HInstruction* org, HInstruction* offset) void HLoopOptimization::GenerateVecMem(HInstruction* org, HInstruction* opa, HInstruction* opb, + HInstruction* offset, Primitive::Type type) { HInstruction* vector = nullptr; if (vector_mode_ == kVector) { // Vector store or load. + HInstruction* base = org->InputAt(0); if (opb != nullptr) { vector = new (global_allocator_) HVecStore( - global_allocator_, org->InputAt(0), opa, opb, type, vector_length_); + global_allocator_, base, opa, opb, type, vector_length_); } else { bool is_string_char_at = org->AsArrayGet()->IsStringCharAt(); vector = new (global_allocator_) HVecLoad( - global_allocator_, org->InputAt(0), opa, type, vector_length_, is_string_char_at); + global_allocator_, base, opa, type, vector_length_, is_string_char_at); + } + // Known dynamically enforced alignment? + // TODO: detect offset + constant differences. + // TODO: long run, static alignment analysis? + if (vector_peeling_candidate_ != nullptr && + vector_peeling_candidate_->base == base && + vector_peeling_candidate_->offset == offset) { + vector->AsVecMemoryOperation()->SetAlignment(Alignment(kAlignedBase, 0)); } } else { // Scalar store or load. @@ -1443,10 +1551,57 @@ bool HLoopOptimization::VectorizeHalvingAddIdiom(LoopNode* node, } // +// Vectorization heuristics. +// + +bool HLoopOptimization::IsVectorizationProfitable(int64_t trip_count) { + // Current heuristic: non-empty body with sufficient number + // of iterations (if known). + // TODO: refine by looking at e.g. operation count, alignment, etc. + if (vector_length_ == 0) { + return false; // nothing found + } else if (0 < trip_count && trip_count < vector_length_) { + return false; // insufficient iterations + } + return true; +} + +void HLoopOptimization::SetPeelingCandidate(int64_t trip_count ATTRIBUTE_UNUSED) { + // Current heuristic: none. + // TODO: implement +} + +uint32_t HLoopOptimization::GetUnrollingFactor(HBasicBlock* block, int64_t trip_count) { + // Current heuristic: unroll by 2 on ARM64/X86 for large known trip + // counts and small loop bodies. + // TODO: refine with operation count, remaining iterations, etc. + // Artem had some really cool ideas for this already. + switch (compiler_driver_->GetInstructionSet()) { + case kArm64: + case kX86: + case kX86_64: { + size_t num_instructions = block->GetInstructions().CountSize(); + if (num_instructions <= 10 && trip_count >= 4 * vector_length_) { + return 2; + } + return 1; + } + default: + return 1; + } +} + +// // Helpers. // bool HLoopOptimization::TrySetPhiInduction(HPhi* phi, bool restrict_uses) { + // Special case Phis that have equivalent in a debuggable setup. Our graph checker isn't + // smart enough to follow strongly connected components (and it's probably not worth + // it to make it so). See b/33775412. + if (graph_->IsDebuggable() && phi->HasEquivalentPhi()) { + return false; + } DCHECK(iset_->empty()); ArenaSet<HInstruction*>* set = induction_range_.LookupCycle(phi); if (set != nullptr) { @@ -1575,8 +1730,8 @@ bool HLoopOptimization::TryReplaceWithLastValue(HLoopInformation* loop_info, size_t index = it->GetIndex(); ++it; // increment before replacing if (iset_->find(user->GetHolder()) == iset_->end()) { // not excluded? - HLoopInformation* other_loop_info = user->GetHolder()->GetBlock()->GetLoopInformation(); // Only update environment uses after the loop. + HLoopInformation* other_loop_info = user->GetHolder()->GetBlock()->GetLoopInformation(); if (other_loop_info == nullptr || !other_loop_info->IsIn(*loop_info)) { user->RemoveAsUserOfInput(index); user->SetRawEnvAt(index, replacement); @@ -1613,4 +1768,21 @@ void HLoopOptimization::RemoveDeadInstructions(const HInstructionList& list) { } } +bool HLoopOptimization::CanRemoveCycle() { + for (HInstruction* i : *iset_) { + // We can never remove instructions that have environment + // uses when we compile 'debuggable'. + if (i->HasEnvironmentUses() && graph_->IsDebuggable()) { + return false; + } + // A deoptimization should never have an environment input removed. + for (const HUseListNode<HEnvironment*>& use : i->GetEnvUses()) { + if (use.GetUser()->GetHolder()->IsDeoptimize()) { + return false; + } + } + } + return true; +} + } // namespace art diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h index cc6343aeb5..de4bd85fc8 100644 --- a/compiler/optimizing/loop_optimization.h +++ b/compiler/optimizing/loop_optimization.h @@ -116,14 +116,15 @@ class HLoopOptimization : public HOptimization { void OptimizeInnerLoop(LoopNode* node); // Vectorization analysis and synthesis. - bool CanVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count); + bool ShouldVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count); void Vectorize(LoopNode* node, HBasicBlock* block, HBasicBlock* exit, int64_t trip_count); void GenerateNewLoop(LoopNode* node, HBasicBlock* block, HBasicBlock* new_preheader, HInstruction* lo, HInstruction* hi, - HInstruction* step); + HInstruction* step, + uint32_t unroll); bool VectorizeDef(LoopNode* node, HInstruction* instruction, bool generate_code); bool VectorizeUse(LoopNode* node, HInstruction* instruction, @@ -133,10 +134,11 @@ class HLoopOptimization : public HOptimization { bool TrySetVectorType(Primitive::Type type, /*out*/ uint64_t* restrictions); bool TrySetVectorLength(uint32_t length); void GenerateVecInv(HInstruction* org, Primitive::Type type); - void GenerateVecSub(HInstruction* org, HInstruction* off); + void GenerateVecSub(HInstruction* org, HInstruction* offset); void GenerateVecMem(HInstruction* org, HInstruction* opa, HInstruction* opb, + HInstruction* offset, Primitive::Type type); void GenerateVecOp(HInstruction* org, HInstruction* opa, @@ -151,6 +153,11 @@ class HLoopOptimization : public HOptimization { Primitive::Type type, uint64_t restrictions); + // Vectorization heuristics. + bool IsVectorizationProfitable(int64_t trip_count); + void SetPeelingCandidate(int64_t trip_count); + uint32_t GetUnrollingFactor(HBasicBlock* block, int64_t trip_count); + // Helpers. bool TrySetPhiInduction(HPhi* phi, bool restrict_uses); bool TrySetSimpleLoopHeader(HBasicBlock* block); @@ -208,20 +215,25 @@ class HLoopOptimization : public HOptimization { // Contents reside in phase-local heap memory. ArenaSet<ArrayReference>* vector_refs_; + // Dynamic loop peeling candidate for alignment. + const ArrayReference* vector_peeling_candidate_; + + // Dynamic data dependence test of the form a != b. + HInstruction* vector_runtime_test_a_; + HInstruction* vector_runtime_test_b_; + // Mapping used during vectorization synthesis for both the scalar peeling/cleanup - // loop (simd_ is false) and the actual vector loop (simd_ is true). The data + // loop (mode is kSequential) and the actual vector loop (mode is kVector). The data // structure maps original instructions into the new instructions. // Contents reside in phase-local heap memory. ArenaSafeMap<HInstruction*, HInstruction*>* vector_map_; // Temporary vectorization bookkeeping. + VectorMode vector_mode_; // synthesis mode HBasicBlock* vector_preheader_; // preheader of the new loop HBasicBlock* vector_header_; // header of the new loop HBasicBlock* vector_body_; // body of the new loop - HInstruction* vector_runtime_test_a_; - HInstruction* vector_runtime_test_b_; // defines a != b runtime test - HPhi* vector_phi_; // the Phi representing the normalized loop index - VectorMode vector_mode_; // selects synthesis mode + HInstruction* vector_index_; // normalized index of the new loop friend class LoopOptimizationTest; diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 9a91287670..4ca833707b 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -967,6 +967,7 @@ void HInstructionList::AddInstruction(HInstruction* instruction) { DCHECK(last_instruction_ == nullptr); first_instruction_ = last_instruction_ = instruction; } else { + DCHECK(last_instruction_ != nullptr); last_instruction_->next_ = instruction; instruction->previous_ = last_instruction_; last_instruction_ = instruction; @@ -2623,7 +2624,7 @@ const DexFile& HInvokeStaticOrDirect::GetDexFileForPcRelativeDexCache() const { } bool HInvokeStaticOrDirect::NeedsDexCacheOfDeclaringClass() const { - if (GetMethodLoadKind() != MethodLoadKind::kDexCacheViaMethod) { + if (GetMethodLoadKind() != MethodLoadKind::kRuntimeCall) { return false; } if (!IsIntrinsic()) { @@ -2643,10 +2644,10 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::MethodLoadKind return os << "BootImageLinkTimePcRelative"; case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: return os << "DirectAddress"; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - return os << "DexCachePcRelative"; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: - return os << "DexCacheViaMethod"; + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: + return os << "BssEntry"; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: + return os << "RuntimeCall"; default: LOG(FATAL) << "Unknown MethodLoadKind: " << static_cast<int>(rhs); UNREACHABLE(); diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index befd0ff97b..5e072cdb67 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -421,7 +421,7 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { void SimplifyLoop(HBasicBlock* header); int32_t GetNextInstructionId() { - DCHECK_NE(current_instruction_id_, INT32_MAX); + CHECK_NE(current_instruction_id_, INT32_MAX); return current_instruction_id_++; } @@ -430,7 +430,7 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { } void SetCurrentInstructionId(int32_t id) { - DCHECK_GE(id, current_instruction_id_); + CHECK_GE(id, current_instruction_id_); current_instruction_id_ = id; } @@ -1410,12 +1410,7 @@ class HLoopInformationOutwardIterator : public ValueObject { M(IntermediateAddressIndex, Instruction) #endif -#ifndef ART_ENABLE_CODEGEN_arm #define FOR_EACH_CONCRETE_INSTRUCTION_ARM(M) -#else -#define FOR_EACH_CONCRETE_INSTRUCTION_ARM(M) \ - M(ArmDexCacheArraysBase, Instruction) -#endif #define FOR_EACH_CONCRETE_INSTRUCTION_ARM64(M) @@ -1424,7 +1419,6 @@ class HLoopInformationOutwardIterator : public ValueObject { #else #define FOR_EACH_CONCRETE_INSTRUCTION_MIPS(M) \ M(MipsComputeBaseMethodAddress, Instruction) \ - M(MipsDexCacheArraysBase, Instruction) \ M(MipsPackedSwitch, Instruction) #endif @@ -1485,7 +1479,8 @@ FOR_EACH_INSTRUCTION(FORWARD_DECLARATION) H##type* As##type() { return this; } template <typename T> -class HUseListNode : public ArenaObject<kArenaAllocUseListNode> { +class HUseListNode : public ArenaObject<kArenaAllocUseListNode>, + public IntrusiveForwardListNode<HUseListNode<T>> { public: // Get the instruction which has this use as one of the inputs. T GetUser() const { return user_; } @@ -1494,10 +1489,6 @@ class HUseListNode : public ArenaObject<kArenaAllocUseListNode> { // Set the position of the input record that this use corresponds to. void SetIndex(size_t index) { index_ = index; } - // Hook for the IntrusiveForwardList<>. - // TODO: Hide this better. - IntrusiveForwardListHook hook; - private: HUseListNode(T user, size_t index) : user_(user), index_(index) {} @@ -2621,6 +2612,16 @@ class HPhi FINAL : public HVariableInputSizeInstruction { && other->AsPhi()->GetRegNumber() == GetRegNumber(); } + bool HasEquivalentPhi() const { + if (GetPrevious() != nullptr && GetPrevious()->AsPhi()->GetRegNumber() == GetRegNumber()) { + return true; + } + if (GetNext() != nullptr && GetNext()->AsPhi()->GetRegNumber() == GetRegNumber()) { + return true; + } + return false; + } + // Returns the next equivalent phi (starting from the current one) or null if there is none. // An equivalent phi is a phi having the same dex register and type. // It assumes that phis with the same dex register are adjacent. @@ -4166,17 +4167,13 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // Used for app->boot calls with non-relocatable image and for JIT-compiled calls. kDirectAddress, - // Load from resolved methods array in the dex cache using a PC-relative load. - // Used when we need to use the dex cache, for example for invoke-static that - // may cause class initialization (the entry may point to a resolution method), - // and we know that we can access the dex cache arrays using a PC-relative load. - kDexCachePcRelative, - - // Use ArtMethod* from the resolved methods of the compiled method's own ArtMethod*. - // Used for JIT when we need to use the dex cache. This is also the last-resort-kind - // used when other kinds are unavailable (say, dex cache arrays are not PC-relative) - // or unimplemented or impractical (i.e. slow) on a particular architecture. - kDexCacheViaMethod, + // Load from an entry in the .bss section using a PC-relative load. + // Used for classes outside boot image when .bss is accessible with a PC-relative load. + kBssEntry, + + // Make a runtime call to resolve and call the method. This is the last-resort-kind + // used when other kinds are unimplemented on a particular architecture. + kRuntimeCall, }; // Determines the location of the code pointer. @@ -4197,7 +4194,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // - thread entrypoint offset for kStringInit method if this is a string init invoke. // Note that there are multiple string init methods, each having its own offset. // - the method address for kDirectAddress - // - the dex cache arrays offset for kDexCachePcRel. uint64_t method_load_data; }; @@ -4298,12 +4294,9 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { bool NeedsDexCacheOfDeclaringClass() const OVERRIDE; bool IsStringInit() const { return GetMethodLoadKind() == MethodLoadKind::kStringInit; } bool HasMethodAddress() const { return GetMethodLoadKind() == MethodLoadKind::kDirectAddress; } - bool HasPcRelativeDexCache() const { - return GetMethodLoadKind() == MethodLoadKind::kDexCachePcRelative; - } bool HasPcRelativeMethodLoadKind() const { return GetMethodLoadKind() == MethodLoadKind::kBootImageLinkTimePcRelative || - GetMethodLoadKind() == MethodLoadKind::kDexCachePcRelative; + GetMethodLoadKind() == MethodLoadKind::kBssEntry; } bool HasCurrentMethodInput() const { // This function can be called only after the invoke has been fully initialized by the builder. @@ -4327,11 +4320,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { return dispatch_info_.method_load_data; } - uint32_t GetDexCacheArrayOffset() const { - DCHECK(HasPcRelativeDexCache()); - return dispatch_info_.method_load_data; - } - const DexFile& GetDexFileForPcRelativeDexCache() const; ClinitCheckRequirement GetClinitCheckRequirement() const { @@ -4376,7 +4364,7 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // Does this method load kind need the current method as an input? static bool NeedsCurrentMethodInput(MethodLoadKind kind) { - return kind == MethodLoadKind::kRecursive || kind == MethodLoadKind::kDexCacheViaMethod; + return kind == MethodLoadKind::kRecursive || kind == MethodLoadKind::kRuntimeCall; } DECLARE_INSTRUCTION(InvokeStaticOrDirect); @@ -6881,9 +6869,6 @@ class HParallelMove FINAL : public HTemplateInstruction<0> { #if defined(ART_ENABLE_CODEGEN_arm) || defined(ART_ENABLE_CODEGEN_arm64) #include "nodes_shared.h" #endif -#ifdef ART_ENABLE_CODEGEN_arm -#include "nodes_arm.h" -#endif #ifdef ART_ENABLE_CODEGEN_mips #include "nodes_mips.h" #endif diff --git a/compiler/optimizing/nodes_arm.h b/compiler/optimizing/nodes_arm.h deleted file mode 100644 index d9f9740e73..0000000000 --- a/compiler/optimizing/nodes_arm.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_NODES_ARM_H_ -#define ART_COMPILER_OPTIMIZING_NODES_ARM_H_ - -namespace art { - -class HArmDexCacheArraysBase FINAL : public HExpression<0> { - public: - explicit HArmDexCacheArraysBase(const DexFile& dex_file) - : HExpression(Primitive::kPrimInt, SideEffects::None(), kNoDexPc), - dex_file_(&dex_file), - element_offset_(static_cast<size_t>(-1)) { } - - bool CanBeMoved() const OVERRIDE { return true; } - - void UpdateElementOffset(size_t element_offset) { - // Use the lowest offset from the requested elements so that all offsets from - // this base are non-negative because our assemblers emit negative-offset loads - // as a sequence of two or more instructions. (However, positive offsets beyond - // 4KiB also require two or more instructions, so this simple heuristic could - // be improved for cases where there is a dense cluster of elements far from - // the lowest offset. This is expected to be rare enough though, so we choose - // not to spend compile time on elaborate calculations.) - element_offset_ = std::min(element_offset_, element_offset); - } - - const DexFile& GetDexFile() const { - return *dex_file_; - } - - size_t GetElementOffset() const { - return element_offset_; - } - - DECLARE_INSTRUCTION(ArmDexCacheArraysBase); - - private: - const DexFile* dex_file_; - size_t element_offset_; - - DISALLOW_COPY_AND_ASSIGN(HArmDexCacheArraysBase); -}; - -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_NODES_ARM_H_ diff --git a/compiler/optimizing/nodes_mips.h b/compiler/optimizing/nodes_mips.h index 36431c1fb9..8e439d9621 100644 --- a/compiler/optimizing/nodes_mips.h +++ b/compiler/optimizing/nodes_mips.h @@ -34,38 +34,6 @@ class HMipsComputeBaseMethodAddress : public HExpression<0> { DISALLOW_COPY_AND_ASSIGN(HMipsComputeBaseMethodAddress); }; -class HMipsDexCacheArraysBase : public HExpression<0> { - public: - explicit HMipsDexCacheArraysBase(const DexFile& dex_file) - : HExpression(Primitive::kPrimInt, SideEffects::None(), kNoDexPc), - dex_file_(&dex_file), - element_offset_(static_cast<size_t>(-1)) { } - - bool CanBeMoved() const OVERRIDE { return true; } - - void UpdateElementOffset(size_t element_offset) { - // We'll maximize the range of a single load instruction for dex cache array accesses - // by aligning offset -32768 with the offset of the first used element. - element_offset_ = std::min(element_offset_, element_offset); - } - - const DexFile& GetDexFile() const { - return *dex_file_; - } - - size_t GetElementOffset() const { - return element_offset_; - } - - DECLARE_INSTRUCTION(MipsDexCacheArraysBase); - - private: - const DexFile* dex_file_; - size_t element_offset_; - - DISALLOW_COPY_AND_ASSIGN(HMipsDexCacheArraysBase); -}; - // Mips version of HPackedSwitch that holds a pointer to the base method address. class HMipsPackedSwitch FINAL : public HTemplateInstruction<2> { public: diff --git a/compiler/optimizing/nodes_vector.h b/compiler/optimizing/nodes_vector.h index 5dbe29b4fa..6261171a00 100644 --- a/compiler/optimizing/nodes_vector.h +++ b/compiler/optimizing/nodes_vector.h @@ -46,6 +46,10 @@ class Alignment { return "ALIGN(" + std::to_string(base_) + "," + std::to_string(offset_) + ")"; } + bool operator==(const Alignment& other) const { + return base_ == other.base_ && offset_ == other.offset_; + } + private: size_t base_; size_t offset_; @@ -96,6 +100,19 @@ class HVecOperation : public HVariableInputSizeInstruction { return GetPackedField<TypeField>(); } + // Assumes vector nodes cannot be moved by default. Each concrete implementation + // that can be moved should override this method and return true. + bool CanBeMoved() const OVERRIDE { return false; } + + // Tests if all data of a vector node (vector length and packed type) is equal. + // Each concrete implementation that adds more fields should test equality of + // those fields in its own method *and* call all super methods. + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecOperation()); + const HVecOperation* o = other->AsVecOperation(); + return GetVectorLength() == o->GetVectorLength() && GetPackedType() == o->GetPackedType(); + } + DECLARE_ABSTRACT_INSTRUCTION(VecOperation); protected: @@ -189,6 +206,12 @@ class HVecMemoryOperation : public HVecOperation { HInstruction* GetArray() const { return InputAt(0); } HInstruction* GetIndex() const { return InputAt(1); } + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecMemoryOperation()); + const HVecMemoryOperation* o = other->AsVecMemoryOperation(); + return HVecOperation::InstructionDataEquals(o) && GetAlignment() == o->GetAlignment(); + } + DECLARE_ABSTRACT_INSTRUCTION(VecMemoryOperation); private: @@ -231,7 +254,13 @@ class HVecReplicateScalar FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, scalar, packed_type, vector_length, dex_pc) { DCHECK(!scalar->IsVecOperation()); } + + // A replicate needs to stay in place, since SIMD registers are not + // kept alive across vector loop boundaries (yet). + bool CanBeMoved() const OVERRIDE { return false; } + DECLARE_INSTRUCTION(VecReplicateScalar); + private: DISALLOW_COPY_AND_ASSIGN(HVecReplicateScalar); }; @@ -251,7 +280,10 @@ class HVecSumReduce FINAL : public HVecUnaryOperation { // TODO: probably integral promotion Primitive::Type GetType() const OVERRIDE { return GetPackedType(); } + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecSumReduce); + private: DISALLOW_COPY_AND_ASSIGN(HVecSumReduce); }; @@ -273,6 +305,8 @@ class HVecCnv FINAL : public HVecUnaryOperation { Primitive::Type GetInputType() const { return InputAt(0)->AsVecOperation()->GetPackedType(); } Primitive::Type GetResultType() const { return GetPackedType(); } + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecCnv); private: @@ -291,7 +325,11 @@ class HVecNeg FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, input, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(input, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecNeg); + private: DISALLOW_COPY_AND_ASSIGN(HVecNeg); }; @@ -308,7 +346,11 @@ class HVecAbs FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, input, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(input, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAbs); + private: DISALLOW_COPY_AND_ASSIGN(HVecAbs); }; @@ -326,7 +368,11 @@ class HVecNot FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, input, packed_type, vector_length, dex_pc) { DCHECK(input->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecNot); + private: DISALLOW_COPY_AND_ASSIGN(HVecNot); }; @@ -349,7 +395,11 @@ class HVecAdd FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAdd); + private: DISALLOW_COPY_AND_ASSIGN(HVecAdd); }; @@ -378,6 +428,16 @@ class HVecHalvingAdd FINAL : public HVecBinaryOperation { bool IsUnsigned() const { return GetPackedFlag<kFieldHAddIsUnsigned>(); } bool IsRounded() const { return GetPackedFlag<kFieldHAddIsRounded>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecHalvingAdd()); + const HVecHalvingAdd* o = other->AsVecHalvingAdd(); + return HVecOperation::InstructionDataEquals(o) && + IsUnsigned() == o->IsUnsigned() && + IsRounded() == o->IsRounded(); + } + DECLARE_INSTRUCTION(VecHalvingAdd); private: @@ -404,7 +464,11 @@ class HVecSub FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecSub); + private: DISALLOW_COPY_AND_ASSIGN(HVecSub); }; @@ -423,7 +487,11 @@ class HVecMul FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecMul); + private: DISALLOW_COPY_AND_ASSIGN(HVecMul); }; @@ -442,7 +510,11 @@ class HVecDiv FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecDiv); + private: DISALLOW_COPY_AND_ASSIGN(HVecDiv); }; @@ -466,6 +538,14 @@ class HVecMin FINAL : public HVecBinaryOperation { bool IsUnsigned() const { return GetPackedFlag<kFieldMinOpIsUnsigned>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecMin()); + const HVecMin* o = other->AsVecMin(); + return HVecOperation::InstructionDataEquals(o) && IsUnsigned() == o->IsUnsigned(); + } + DECLARE_INSTRUCTION(VecMin); private: @@ -496,6 +576,14 @@ class HVecMax FINAL : public HVecBinaryOperation { bool IsUnsigned() const { return GetPackedFlag<kFieldMaxOpIsUnsigned>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecMax()); + const HVecMax* o = other->AsVecMax(); + return HVecOperation::InstructionDataEquals(o) && IsUnsigned() == o->IsUnsigned(); + } + DECLARE_INSTRUCTION(VecMax); private: @@ -520,7 +608,11 @@ class HVecAnd FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAnd); + private: DISALLOW_COPY_AND_ASSIGN(HVecAnd); }; @@ -538,7 +630,11 @@ class HVecAndNot FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAndNot); + private: DISALLOW_COPY_AND_ASSIGN(HVecAndNot); }; @@ -556,7 +652,11 @@ class HVecOr FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecOr); + private: DISALLOW_COPY_AND_ASSIGN(HVecOr); }; @@ -574,7 +674,11 @@ class HVecXor FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecXor); + private: DISALLOW_COPY_AND_ASSIGN(HVecXor); }; @@ -592,7 +696,11 @@ class HVecShl FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(left, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecShl); + private: DISALLOW_COPY_AND_ASSIGN(HVecShl); }; @@ -610,7 +718,11 @@ class HVecShr FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(left, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecShr); + private: DISALLOW_COPY_AND_ASSIGN(HVecShr); }; @@ -628,7 +740,11 @@ class HVecUShr FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(left, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecUShr); + private: DISALLOW_COPY_AND_ASSIGN(HVecUShr); }; @@ -656,7 +772,13 @@ class HVecSetScalars FINAL : public HVecOperation { SetRawInputAt(0, scalars[i]); } } + + // Setting scalars needs to stay in place, since SIMD registers are not + // kept alive across vector loop boundaries (yet). + bool CanBeMoved() const OVERRIDE { return false; } + DECLARE_INSTRUCTION(VecSetScalars); + private: DISALLOW_COPY_AND_ASSIGN(HVecSetScalars); }; @@ -697,7 +819,9 @@ class HVecMultiplyAccumulate FINAL : public HVecOperation { bool CanBeMoved() const OVERRIDE { return true; } bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { - return op_kind_ == other->AsVecMultiplyAccumulate()->op_kind_; + DCHECK(other->IsVecMultiplyAccumulate()); + const HVecMultiplyAccumulate* o = other->AsVecMultiplyAccumulate(); + return HVecOperation::InstructionDataEquals(o) && GetOpKind() == o->GetOpKind(); } InstructionKind GetOpKind() const { return op_kind_; } @@ -732,10 +856,19 @@ class HVecLoad FINAL : public HVecMemoryOperation { SetRawInputAt(1, index); SetPackedFlag<kFieldIsStringCharAt>(is_string_char_at); } - DECLARE_INSTRUCTION(VecLoad); bool IsStringCharAt() const { return GetPackedFlag<kFieldIsStringCharAt>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecLoad()); + const HVecLoad* o = other->AsVecLoad(); + return HVecMemoryOperation::InstructionDataEquals(o) && IsStringCharAt() == o->IsStringCharAt(); + } + + DECLARE_INSTRUCTION(VecLoad); + private: // Additional packed bits. static constexpr size_t kFieldIsStringCharAt = HVecOperation::kNumberOfVectorOpPackedBits; @@ -767,7 +900,12 @@ class HVecStore FINAL : public HVecMemoryOperation { SetRawInputAt(1, index); SetRawInputAt(2, value); } + + // A store needs to stay in place. + bool CanBeMoved() const OVERRIDE { return false; } + DECLARE_INSTRUCTION(VecStore); + private: DISALLOW_COPY_AND_ASSIGN(HVecStore); }; diff --git a/compiler/optimizing/nodes_vector_test.cc b/compiler/optimizing/nodes_vector_test.cc new file mode 100644 index 0000000000..0238ea4602 --- /dev/null +++ b/compiler/optimizing/nodes_vector_test.cc @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "base/arena_allocator.h" +#include "nodes.h" +#include "optimizing_unit_test.h" + +namespace art { + +/** + * Fixture class for testing vector nodes. + */ +class NodesVectorTest : public CommonCompilerTest { + public: + NodesVectorTest() + : pool_(), + allocator_(&pool_), + graph_(CreateGraph(&allocator_)) { + BuildGraph(); + } + + ~NodesVectorTest() { } + + void BuildGraph() { + graph_->SetNumberOfVRegs(1); + entry_block_ = new (&allocator_) HBasicBlock(graph_); + exit_block_ = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(entry_block_); + graph_->AddBlock(exit_block_); + graph_->SetEntryBlock(entry_block_); + graph_->SetExitBlock(exit_block_); + parameter_ = new (&allocator_) HParameterValue(graph_->GetDexFile(), + dex::TypeIndex(0), + 0, + Primitive::kPrimInt); + entry_block_->AddInstruction(parameter_); + } + + // General building fields. + ArenaPool pool_; + ArenaAllocator allocator_; + HGraph* graph_; + + HBasicBlock* entry_block_; + HBasicBlock* exit_block_; + + HInstruction* parameter_; +}; + +// +// The actual vector nodes tests. +// + +TEST(NodesVector, Alignment) { + EXPECT_TRUE(Alignment(1, 0).IsAlignedAt(1)); + EXPECT_FALSE(Alignment(1, 0).IsAlignedAt(2)); + + EXPECT_TRUE(Alignment(2, 0).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(2, 1).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(2, 0).IsAlignedAt(2)); + EXPECT_FALSE(Alignment(2, 1).IsAlignedAt(2)); + EXPECT_FALSE(Alignment(2, 0).IsAlignedAt(4)); + EXPECT_FALSE(Alignment(2, 1).IsAlignedAt(4)); + + EXPECT_TRUE(Alignment(4, 0).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(4, 2).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(4, 0).IsAlignedAt(2)); + EXPECT_TRUE(Alignment(4, 2).IsAlignedAt(2)); + EXPECT_TRUE(Alignment(4, 0).IsAlignedAt(4)); + EXPECT_FALSE(Alignment(4, 2).IsAlignedAt(4)); + EXPECT_FALSE(Alignment(4, 0).IsAlignedAt(8)); + EXPECT_FALSE(Alignment(4, 2).IsAlignedAt(8)); + + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(2)); + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(4)); + EXPECT_TRUE(Alignment(16, 8).IsAlignedAt(8)); + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(16)); + EXPECT_FALSE(Alignment(16, 1).IsAlignedAt(16)); + EXPECT_FALSE(Alignment(16, 7).IsAlignedAt(16)); + EXPECT_FALSE(Alignment(16, 0).IsAlignedAt(32)); +} + +TEST(NodesVector, AlignmentEQ) { + EXPECT_TRUE(Alignment(2, 0) == Alignment(2, 0)); + EXPECT_TRUE(Alignment(2, 1) == Alignment(2, 1)); + EXPECT_TRUE(Alignment(4, 0) == Alignment(4, 0)); + EXPECT_TRUE(Alignment(4, 2) == Alignment(4, 2)); + + EXPECT_FALSE(Alignment(4, 0) == Alignment(2, 0)); + EXPECT_FALSE(Alignment(4, 0) == Alignment(4, 1)); + EXPECT_FALSE(Alignment(4, 0) == Alignment(8, 0)); +} + +TEST(NodesVector, AlignmentString) { + EXPECT_STREQ("ALIGN(1,0)", Alignment(1, 0).ToString().c_str()); + + EXPECT_STREQ("ALIGN(2,0)", Alignment(2, 0).ToString().c_str()); + EXPECT_STREQ("ALIGN(2,1)", Alignment(2, 1).ToString().c_str()); + + EXPECT_STREQ("ALIGN(16,0)", Alignment(16, 0).ToString().c_str()); + EXPECT_STREQ("ALIGN(16,1)", Alignment(16, 1).ToString().c_str()); + EXPECT_STREQ("ALIGN(16,8)", Alignment(16, 8).ToString().c_str()); + EXPECT_STREQ("ALIGN(16,9)", Alignment(16, 9).ToString().c_str()); +} + +TEST_F(NodesVectorTest, VectorOperationProperties) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + HVecOperation* v1 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + HVecOperation* v2 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 2); + HVecOperation* v3 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimShort, 4); + HVecOperation* v4 = new (&allocator_) + HVecStore(&allocator_, parameter_, parameter_, v0, Primitive::kPrimInt, 4); + + EXPECT_TRUE(v0->Equals(v0)); + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + EXPECT_TRUE(v4->Equals(v4)); + + EXPECT_TRUE(v0->Equals(v1)); + EXPECT_FALSE(v0->Equals(v2)); // different vector lengths + EXPECT_FALSE(v0->Equals(v3)); // different packed types + EXPECT_FALSE(v0->Equals(v4)); // different kinds + + EXPECT_TRUE(v1->Equals(v0)); // switch operands + EXPECT_FALSE(v4->Equals(v0)); + + EXPECT_EQ(4u, v0->GetVectorLength()); + EXPECT_EQ(4u, v1->GetVectorLength()); + EXPECT_EQ(2u, v2->GetVectorLength()); + EXPECT_EQ(4u, v3->GetVectorLength()); + EXPECT_EQ(4u, v4->GetVectorLength()); + + EXPECT_EQ(Primitive::kPrimDouble, v0->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v1->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v2->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v3->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v4->GetType()); + + EXPECT_EQ(Primitive::kPrimInt, v0->GetPackedType()); + EXPECT_EQ(Primitive::kPrimInt, v1->GetPackedType()); + EXPECT_EQ(Primitive::kPrimInt, v2->GetPackedType()); + EXPECT_EQ(Primitive::kPrimShort, v3->GetPackedType()); + EXPECT_EQ(Primitive::kPrimInt, v4->GetPackedType()); + + EXPECT_EQ(16u, v0->GetVectorNumberOfBytes()); + EXPECT_EQ(16u, v1->GetVectorNumberOfBytes()); + EXPECT_EQ(8u, v2->GetVectorNumberOfBytes()); + EXPECT_EQ(8u, v3->GetVectorNumberOfBytes()); + EXPECT_EQ(16u, v4->GetVectorNumberOfBytes()); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_FALSE(v1->CanBeMoved()); + EXPECT_FALSE(v2->CanBeMoved()); + EXPECT_FALSE(v3->CanBeMoved()); + EXPECT_FALSE(v4->CanBeMoved()); +} + +TEST_F(NodesVectorTest, VectorAlignmentAndStringCharAtMatterOnLoad) { + HVecLoad* v0 = new (&allocator_) + HVecLoad(&allocator_, parameter_, parameter_, Primitive::kPrimInt, 4, /*is_string_char_at*/ false); + HVecLoad* v1 = new (&allocator_) + HVecLoad(&allocator_, parameter_, parameter_, Primitive::kPrimInt, 4, /*is_string_char_at*/ false); + HVecLoad* v2 = new (&allocator_) + HVecLoad(&allocator_, parameter_, parameter_, Primitive::kPrimInt, 4, /*is_string_char_at*/ true); + + EXPECT_TRUE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + + EXPECT_FALSE(v0->IsStringCharAt()); + EXPECT_FALSE(v1->IsStringCharAt()); + EXPECT_TRUE(v2->IsStringCharAt()); + + EXPECT_TRUE(v0->Equals(v0)); + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + + EXPECT_TRUE(v0->Equals(v1)); + EXPECT_FALSE(v0->Equals(v2)); + + EXPECT_TRUE(v0->GetAlignment() == Alignment(4, 0)); + EXPECT_TRUE(v1->GetAlignment() == Alignment(4, 0)); + EXPECT_TRUE(v2->GetAlignment() == Alignment(4, 0)); + + v1->SetAlignment(Alignment(8, 0)); + + EXPECT_TRUE(v1->GetAlignment() == Alignment(8, 0)); + + EXPECT_FALSE(v0->Equals(v1)); // no longer equal +} + +TEST_F(NodesVectorTest, VectorSignMattersOnMin) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecMin* v1 = new (&allocator_) + HVecMin(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true); + HVecMin* v2 = new (&allocator_) + HVecMin(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false); + HVecMin* v3 = new (&allocator_) + HVecMin(&allocator_, v0, v0, Primitive::kPrimInt, 2, /*is_unsigned*/ true); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + + EXPECT_TRUE(v1->IsUnsigned()); + EXPECT_FALSE(v2->IsUnsigned()); + EXPECT_TRUE(v3->IsUnsigned()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + + EXPECT_FALSE(v1->Equals(v2)); // different signs + EXPECT_FALSE(v1->Equals(v3)); // different vector lengths +} + +TEST_F(NodesVectorTest, VectorSignMattersOnMax) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecMax* v1 = new (&allocator_) + HVecMax(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true); + HVecMax* v2 = new (&allocator_) + HVecMax(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false); + HVecMax* v3 = new (&allocator_) + HVecMax(&allocator_, v0, v0, Primitive::kPrimInt, 2, /*is_unsigned*/ true); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + + EXPECT_TRUE(v1->IsUnsigned()); + EXPECT_FALSE(v2->IsUnsigned()); + EXPECT_TRUE(v3->IsUnsigned()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + + EXPECT_FALSE(v1->Equals(v2)); // different signs + EXPECT_FALSE(v1->Equals(v3)); // different vector lengths +} + +TEST_F(NodesVectorTest, VectorAttributesMatterOnHalvingAdd) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecHalvingAdd* v1 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true, /*is_rounded*/ true); + HVecHalvingAdd* v2 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true, /*is_rounded*/ false); + HVecHalvingAdd* v3 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false, /*is_rounded*/ true); + HVecHalvingAdd* v4 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false, /*is_rounded*/ false); + HVecHalvingAdd* v5 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 2, /*is_unsigned*/ true, /*is_rounded*/ true); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + EXPECT_TRUE(v4->CanBeMoved()); + EXPECT_TRUE(v5->CanBeMoved()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + EXPECT_TRUE(v4->Equals(v4)); + EXPECT_TRUE(v5->Equals(v5)); + + EXPECT_TRUE(v1->IsUnsigned() && v1->IsRounded()); + EXPECT_TRUE(v2->IsUnsigned() && !v2->IsRounded()); + EXPECT_TRUE(!v3->IsUnsigned() && v3->IsRounded()); + EXPECT_TRUE(!v4->IsUnsigned() && !v4->IsRounded()); + EXPECT_TRUE(v5->IsUnsigned() && v5->IsRounded()); + + EXPECT_FALSE(v1->Equals(v2)); // different attributes + EXPECT_FALSE(v1->Equals(v3)); // different attributes + EXPECT_FALSE(v1->Equals(v4)); // different attributes + EXPECT_FALSE(v1->Equals(v5)); // different vector lengths +} + +TEST_F(NodesVectorTest, VectorOperationMattersOnMultiplyAccumulate) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecMultiplyAccumulate* v1 = new (&allocator_) + HVecMultiplyAccumulate(&allocator_, HInstruction::kAdd, v0, v0, v0, Primitive::kPrimInt, 4); + HVecMultiplyAccumulate* v2 = new (&allocator_) + HVecMultiplyAccumulate(&allocator_, HInstruction::kSub, v0, v0, v0, Primitive::kPrimInt, 4); + HVecMultiplyAccumulate* v3 = new (&allocator_) + HVecMultiplyAccumulate(&allocator_, HInstruction::kAdd, v0, v0, v0, Primitive::kPrimInt, 2); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + + EXPECT_EQ(HInstruction::kAdd, v1->GetOpKind()); + EXPECT_EQ(HInstruction::kSub, v2->GetOpKind()); + EXPECT_EQ(HInstruction::kAdd, v3->GetOpKind()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + + EXPECT_FALSE(v1->Equals(v2)); // different operators + EXPECT_FALSE(v1->Equals(v3)); // different vector lengths +} + +} // namespace art diff --git a/compiler/optimizing/optimizing_cfi_test.cc b/compiler/optimizing/optimizing_cfi_test.cc index 490e50cb77..6cb27b3b1b 100644 --- a/compiler/optimizing/optimizing_cfi_test.cc +++ b/compiler/optimizing/optimizing_cfi_test.cc @@ -24,21 +24,15 @@ #include "optimizing/code_generator.h" #include "optimizing/optimizing_unit_test.h" #include "utils/assembler.h" -#ifdef ART_USE_OLD_ARM_BACKEND -#include "utils/arm/assembler_thumb2.h" -#else #include "utils/arm/assembler_arm_vixl.h" -#endif #include "utils/mips/assembler_mips.h" #include "utils/mips64/assembler_mips64.h" #include "optimizing/optimizing_cfi_test_expected.inc" -#ifndef ART_USE_OLD_ARM_BACKEND namespace vixl32 = vixl::aarch32; using vixl32::r0; -#endif namespace art { @@ -171,18 +165,31 @@ class OptimizingCFITest : public CFITest { #ifdef ART_ENABLE_CODEGEN_arm TEST_ISA(kThumb2) #endif + #ifdef ART_ENABLE_CODEGEN_arm64 +// Run the tests for ARM64 only with Baker read barriers, as the +// expected generated code saves and restore X21 and X22 (instead of +// X20 and X21), as X20 is used as Marking Register in the Baker read +// barrier configuration, and as such is removed from the set of +// callee-save registers in the ARM64 code generator of the Optimizing +// compiler. +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) TEST_ISA(kArm64) #endif +#endif + #ifdef ART_ENABLE_CODEGEN_x86 TEST_ISA(kX86) #endif + #ifdef ART_ENABLE_CODEGEN_x86_64 TEST_ISA(kX86_64) #endif + #ifdef ART_ENABLE_CODEGEN_mips TEST_ISA(kMips) #endif + #ifdef ART_ENABLE_CODEGEN_mips64 TEST_ISA(kMips64) #endif @@ -196,15 +203,6 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { expected_cfi_kThumb2_adjust, expected_cfi_kThumb2_adjust + arraysize(expected_cfi_kThumb2_adjust)); SetUpFrame(kThumb2); -#ifdef ART_USE_OLD_ARM_BACKEND -#define __ down_cast<arm::Thumb2Assembler*>(GetCodeGenerator()->GetAssembler())-> - Label target; - __ CompareAndBranchIfZero(arm::R0, &target); - // Push the target out of range of CBZ. - for (size_t i = 0; i != 65; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } -#else #define __ down_cast<arm::ArmVIXLAssembler*>(GetCodeGenerator() \ ->GetAssembler())->GetVIXLAssembler()-> vixl32::Label target; @@ -213,7 +211,6 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { for (size_t i = 0; i != 65; ++i) { __ Ldr(r0, vixl32::MemOperand(r0)); } -#endif __ Bind(&target); #undef __ Finish(); diff --git a/compiler/optimizing/optimizing_cfi_test_expected.inc b/compiler/optimizing/optimizing_cfi_test_expected.inc index 60af2b4201..77a63acd18 100644 --- a/compiler/optimizing/optimizing_cfi_test_expected.inc +++ b/compiler/optimizing/optimizing_cfi_test_expected.inc @@ -31,21 +31,21 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000010: .cfi_def_cfa_offset: 64 static constexpr uint8_t expected_asm_kArm64[] = { - 0xFF, 0x03, 0x01, 0xD1, 0xF4, 0x17, 0x00, 0xF9, 0xF5, 0x7B, 0x03, 0xA9, - 0xE8, 0xA7, 0x01, 0x6D, 0xE8, 0xA7, 0x41, 0x6D, 0xF4, 0x17, 0x40, 0xF9, - 0xF5, 0x7B, 0x43, 0xA9, 0xFF, 0x03, 0x01, 0x91, 0xC0, 0x03, 0x5F, 0xD6, + 0xFF, 0x03, 0x01, 0xD1, 0xF5, 0x17, 0x00, 0xF9, 0xF6, 0x7B, 0x03, 0xA9, + 0xE8, 0xA7, 0x01, 0x6D, 0xE8, 0xA7, 0x41, 0x6D, 0xF5, 0x17, 0x40, 0xF9, + 0xF6, 0x7B, 0x43, 0xA9, 0xFF, 0x03, 0x01, 0x91, 0xC0, 0x03, 0x5F, 0xD6, }; static constexpr uint8_t expected_cfi_kArm64[] = { - 0x44, 0x0E, 0x40, 0x44, 0x94, 0x06, 0x44, 0x95, 0x04, 0x9E, 0x02, 0x44, + 0x44, 0x0E, 0x40, 0x44, 0x95, 0x06, 0x44, 0x96, 0x04, 0x9E, 0x02, 0x44, 0x05, 0x48, 0x0A, 0x05, 0x49, 0x08, 0x0A, 0x44, 0x06, 0x48, 0x06, 0x49, - 0x44, 0xD4, 0x44, 0xD5, 0xDE, 0x44, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0x40, + 0x44, 0xD5, 0x44, 0xD6, 0xDE, 0x44, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0x40, }; // 0x00000000: sub sp, sp, #0x40 (64) // 0x00000004: .cfi_def_cfa_offset: 64 -// 0x00000004: str x20, [sp, #40] -// 0x00000008: .cfi_offset: r20 at cfa-24 -// 0x00000008: stp x21, lr, [sp, #48] -// 0x0000000c: .cfi_offset: r21 at cfa-16 +// 0x00000004: str x21, [sp, #40] +// 0x00000008: .cfi_offset: r21 at cfa-24 +// 0x00000008: stp x22, lr, [sp, #48] +// 0x0000000c: .cfi_offset: r22 at cfa-16 // 0x0000000c: .cfi_offset: r30 at cfa-8 // 0x0000000c: stp d8, d9, [sp, #24] // 0x00000010: .cfi_offset_extended: r72 at cfa-40 @@ -54,10 +54,10 @@ static constexpr uint8_t expected_cfi_kArm64[] = { // 0x00000010: ldp d8, d9, [sp, #24] // 0x00000014: .cfi_restore_extended: r72 // 0x00000014: .cfi_restore_extended: r73 -// 0x00000014: ldr x20, [sp, #40] -// 0x00000018: .cfi_restore: r20 -// 0x00000018: ldp x21, lr, [sp, #48] -// 0x0000001c: .cfi_restore: r21 +// 0x00000014: ldr x21, [sp, #40] +// 0x00000018: .cfi_restore: r21 +// 0x00000018: ldp x22, lr, [sp, #48] +// 0x0000001c: .cfi_restore: r22 // 0x0000001c: .cfi_restore: r30 // 0x0000001c: add sp, sp, #0x40 (64) // 0x00000020: .cfi_def_cfa_offset: 0 @@ -215,16 +215,11 @@ static constexpr uint8_t expected_cfi_kMips64[] = { // 0x00000034: .cfi_def_cfa_offset: 64 static constexpr uint8_t expected_asm_kThumb2_adjust[] = { -#ifdef ART_USE_OLD_ARM_BACKEND - 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, - 0x40, 0xD0, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, -#else // VIXL emits an extra 2 bytes here for a 32-bit beq as there is no // optimistic 16-bit emit and subsequent fixup for out of reach targets // as with the old assembler. 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, 0x00, 0xF0, 0x41, 0x80, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, -#endif 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, @@ -239,11 +234,7 @@ static constexpr uint8_t expected_asm_kThumb2_adjust[] = { }; static constexpr uint8_t expected_cfi_kThumb2_adjust[] = { 0x42, 0x0E, 0x0C, 0x85, 0x03, 0x86, 0x02, 0x8E, 0x01, 0x44, 0x0E, 0x14, -#ifdef ART_USE_OLD_ARM_BACKEND - 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x86, 0x0A, -#else 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x88, 0x0A, -#endif 0x42, 0x0E, 0x14, 0x44, 0x0E, 0x0C, 0x06, 0x50, 0x06, 0x51, 0x42, 0x0B, 0x0E, 0x40, }; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index e5ab00bce3..b76a0df861 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -24,16 +24,11 @@ #include "android-base/strings.h" -#ifdef ART_ENABLE_CODEGEN_arm -#include "dex_cache_array_fixups_arm.h" -#endif - #ifdef ART_ENABLE_CODEGEN_arm64 #include "instruction_simplifier_arm64.h" #endif #ifdef ART_ENABLE_CODEGEN_mips -#include "dex_cache_array_fixups_mips.h" #include "pc_relative_fixups_mips.h" #endif @@ -81,6 +76,7 @@ #include "jit/debugger_interface.h" #include "jit/jit.h" #include "jit/jit_code_cache.h" +#include "jit/jit_logger.h" #include "jni/quick/jni_compiler.h" #include "licm.h" #include "load_store_analysis.h" @@ -339,7 +335,11 @@ class OptimizingCompiler FINAL : public Compiler { } } - bool JitCompile(Thread* self, jit::JitCodeCache* code_cache, ArtMethod* method, bool osr) + bool JitCompile(Thread* self, + jit::JitCodeCache* code_cache, + ArtMethod* method, + bool osr, + jit::JitLogger* jit_logger) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_); @@ -522,8 +522,6 @@ static HOptimization* BuildOptimization( } else if (opt_name == CodeSinking::kCodeSinkingPassName) { return new (arena) CodeSinking(graph, stats); #ifdef ART_ENABLE_CODEGEN_arm - } else if (opt_name == arm::DexCacheArrayFixups::kDexCacheArrayFixupsArmPassName) { - return new (arena) arm::DexCacheArrayFixups(graph, codegen, stats); } else if (opt_name == arm::InstructionSimplifierArm::kInstructionSimplifierArmPassName) { return new (arena) arm::InstructionSimplifierArm(graph, stats); #endif @@ -532,8 +530,6 @@ static HOptimization* BuildOptimization( return new (arena) arm64::InstructionSimplifierArm64(graph, stats); #endif #ifdef ART_ENABLE_CODEGEN_mips - } else if (opt_name == mips::DexCacheArrayFixups::kDexCacheArrayFixupsMipsPassName) { - return new (arena) mips::DexCacheArrayFixups(graph, codegen, stats); } else if (opt_name == mips::PcRelativeFixups::kPcRelativeFixupsMipsPassName) { return new (arena) mips::PcRelativeFixups(graph, codegen, stats); #endif @@ -641,8 +637,6 @@ void OptimizingCompiler::RunArchOptimizations(InstructionSet instruction_set, #if defined(ART_ENABLE_CODEGEN_arm) case kThumb2: case kArm: { - arm::DexCacheArrayFixups* fixups = - new (arena) arm::DexCacheArrayFixups(graph, codegen, stats); arm::InstructionSimplifierArm* simplifier = new (arena) arm::InstructionSimplifierArm(graph, stats); SideEffectsAnalysis* side_effects = new (arena) SideEffectsAnalysis(graph); @@ -653,7 +647,6 @@ void OptimizingCompiler::RunArchOptimizations(InstructionSet instruction_set, simplifier, side_effects, gvn, - fixups, scheduling, }; RunOptimizations(arm_optimizations, arraysize(arm_optimizations), pass_observer); @@ -682,11 +675,8 @@ void OptimizingCompiler::RunArchOptimizations(InstructionSet instruction_set, case kMips: { mips::PcRelativeFixups* pc_relative_fixups = new (arena) mips::PcRelativeFixups(graph, codegen, stats); - mips::DexCacheArrayFixups* dex_cache_array_fixups = - new (arena) mips::DexCacheArrayFixups(graph, codegen, stats); HOptimization* mips_optimizations[] = { pc_relative_fixups, - dex_cache_array_fixups }; RunOptimizations(mips_optimizations, arraysize(mips_optimizations), pass_observer); break; @@ -1151,7 +1141,8 @@ bool CanEncodeInlinedMethodInStackMap(const DexFile& caller_dex_file, ArtMethod* bool OptimizingCompiler::JitCompile(Thread* self, jit::JitCodeCache* code_cache, ArtMethod* method, - bool osr) { + bool osr, + jit::JitLogger* jit_logger) { StackHandleScope<3> hs(self); Handle<mirror::ClassLoader> class_loader(hs.NewHandle( method->GetDeclaringClass()->GetClassLoader())); @@ -1287,6 +1278,9 @@ bool OptimizingCompiler::JitCompile(Thread* self, } Runtime::Current()->GetJit()->AddMemoryUsage(method, arena.BytesUsed()); + if (jit_logger != nullptr) { + jit_logger->WriteLog(code, code_allocator.GetSize(), method); + } return true; } diff --git a/compiler/optimizing/pc_relative_fixups_mips.cc b/compiler/optimizing/pc_relative_fixups_mips.cc index bce54bf49a..21b645279e 100644 --- a/compiler/optimizing/pc_relative_fixups_mips.cc +++ b/compiler/optimizing/pc_relative_fixups_mips.cc @@ -59,10 +59,9 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { } void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { - // If this is an invoke with PC-relative pointer to a method, + // If this is an invoke with PC-relative load kind, // we need to add the base as the special input. - if (invoke->GetMethodLoadKind() == - HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative && + if (invoke->HasPcRelativeMethodLoadKind() && !IsCallFreeIntrinsic<IntrinsicLocationsBuilderMIPS>(invoke, codegen_)) { InitializePCRelativeBasePointer(); // Add the special argument base to the method. diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc index 98332d35fb..f172e16ff9 100644 --- a/compiler/optimizing/reference_type_propagation.cc +++ b/compiler/optimizing/reference_type_propagation.cc @@ -525,8 +525,8 @@ void ReferenceTypePropagation::RTPVisitor::SetClassAsTypeInfo(HInstruction* inst // Use a null loader. We should probably use the compiling method's class loader, // but then we would need to pass it to RTPVisitor just for this debug check. Since // the method is from the String class, the null loader is good enough. - Handle<mirror::ClassLoader> loader; - ArtMethod* method = cl->ResolveMethod<ClassLinker::kNoICCECheckForCache>( + Handle<mirror::ClassLoader> loader(hs.NewHandle<mirror::ClassLoader>(nullptr)); + ArtMethod* method = cl->ResolveMethod<ClassLinker::ResolveMode::kNoChecks>( dex_file, invoke->GetDexMethodIndex(), dex_cache, loader, nullptr, kDirect); DCHECK(method != nullptr); mirror::Class* declaring_class = method->GetDeclaringClass(); diff --git a/compiler/optimizing/register_allocation_resolver.cc b/compiler/optimizing/register_allocation_resolver.cc index c6a0b6a0d2..ce3a4966aa 100644 --- a/compiler/optimizing/register_allocation_resolver.cc +++ b/compiler/optimizing/register_allocation_resolver.cc @@ -308,8 +308,10 @@ void RegisterAllocationResolver::ConnectSiblings(LiveInterval* interval) { } InsertMoveAfter(interval->GetDefinedBy(), interval->ToLocation(), loc); } - UsePosition* use = current->GetFirstUse(); - EnvUsePosition* env_use = current->GetFirstEnvironmentUse(); + UsePositionList::const_iterator use_it = current->GetUses().begin(); + const UsePositionList::const_iterator use_end = current->GetUses().end(); + EnvUsePositionList::const_iterator env_use_it = current->GetEnvironmentUses().begin(); + const EnvUsePositionList::const_iterator env_use_end = current->GetEnvironmentUses().end(); // Walk over all siblings, updating locations of use positions, and // connecting them when they are adjacent. @@ -321,43 +323,47 @@ void RegisterAllocationResolver::ConnectSiblings(LiveInterval* interval) { LiveRange* range = current->GetFirstRange(); while (range != nullptr) { - while (use != nullptr && use->GetPosition() < range->GetStart()) { - DCHECK(use->IsSynthesized()); - use = use->GetNext(); - } - while (use != nullptr && use->GetPosition() <= range->GetEnd()) { - DCHECK(current->CoversSlow(use->GetPosition()) || (use->GetPosition() == range->GetEnd())); - if (!use->IsSynthesized()) { - LocationSummary* locations = use->GetUser()->GetLocations(); - Location expected_location = locations->InAt(use->GetInputIndex()); + // Process uses in the closed interval [range->GetStart(), range->GetEnd()]. + // FindMatchingUseRange() expects a half-open interval, so pass `range->GetEnd() + 1u`. + size_t range_begin = range->GetStart(); + size_t range_end = range->GetEnd() + 1u; + auto matching_use_range = + FindMatchingUseRange(use_it, use_end, range_begin, range_end); + DCHECK(std::all_of(use_it, + matching_use_range.begin(), + [](const UsePosition& pos) { return pos.IsSynthesized(); })); + for (const UsePosition& use : matching_use_range) { + DCHECK(current->CoversSlow(use.GetPosition()) || (use.GetPosition() == range->GetEnd())); + if (!use.IsSynthesized()) { + LocationSummary* locations = use.GetUser()->GetLocations(); + Location expected_location = locations->InAt(use.GetInputIndex()); // The expected (actual) location may be invalid in case the input is unused. Currently // this only happens for intrinsics. if (expected_location.IsValid()) { if (expected_location.IsUnallocated()) { - locations->SetInAt(use->GetInputIndex(), source); + locations->SetInAt(use.GetInputIndex(), source); } else if (!expected_location.IsConstant()) { - AddInputMoveFor(interval->GetDefinedBy(), use->GetUser(), source, expected_location); + AddInputMoveFor( + interval->GetDefinedBy(), use.GetUser(), source, expected_location); } } else { - DCHECK(use->GetUser()->IsInvoke()); - DCHECK(use->GetUser()->AsInvoke()->GetIntrinsic() != Intrinsics::kNone); + DCHECK(use.GetUser()->IsInvoke()); + DCHECK(use.GetUser()->AsInvoke()->GetIntrinsic() != Intrinsics::kNone); } } - use = use->GetNext(); } + use_it = matching_use_range.end(); // Walk over the environment uses, and update their locations. - while (env_use != nullptr && env_use->GetPosition() < range->GetStart()) { - env_use = env_use->GetNext(); - } - - while (env_use != nullptr && env_use->GetPosition() <= range->GetEnd()) { - DCHECK(current->CoversSlow(env_use->GetPosition()) - || (env_use->GetPosition() == range->GetEnd())); - HEnvironment* environment = env_use->GetEnvironment(); - environment->SetLocationAt(env_use->GetInputIndex(), source); - env_use = env_use->GetNext(); + auto matching_env_use_range = + FindMatchingUseRange(env_use_it, env_use_end, range_begin, range_end); + for (const EnvUsePosition& env_use : matching_env_use_range) { + DCHECK(current->CoversSlow(env_use.GetPosition()) + || (env_use.GetPosition() == range->GetEnd())); + HEnvironment* environment = env_use.GetEnvironment(); + environment->SetLocationAt(env_use.GetInputIndex(), source); } + env_use_it = matching_env_use_range.end(); range = range->GetNext(); } @@ -395,13 +401,8 @@ void RegisterAllocationResolver::ConnectSiblings(LiveInterval* interval) { current = next_sibling; } while (current != nullptr); - if (kIsDebugBuild) { - // Following uses can only be synthesized uses. - while (use != nullptr) { - DCHECK(use->IsSynthesized()); - use = use->GetNext(); - } - } + // Following uses can only be synthesized uses. + DCHECK(std::all_of(use_it, use_end, [](const UsePosition& pos) { return pos.IsSynthesized(); })); } static bool IsMaterializableEntryBlockInstructionOfGraphWithIrreducibleLoop( diff --git a/compiler/optimizing/register_allocator_graph_color.cc b/compiler/optimizing/register_allocator_graph_color.cc index 2fd7b03151..5e22772844 100644 --- a/compiler/optimizing/register_allocator_graph_color.cc +++ b/compiler/optimizing/register_allocator_graph_color.cc @@ -178,18 +178,17 @@ static float ComputeSpillWeight(LiveInterval* interval, const SsaLivenessAnalysi use_weight += CostForMoveAt(interval->GetStart() + 1, liveness); } - UsePosition* use = interval->GetFirstUse(); - while (use != nullptr && use->GetPosition() <= interval->GetStart()) { - // Skip uses before the start of this live interval. - use = use->GetNext(); - } - - while (use != nullptr && use->GetPosition() <= interval->GetEnd()) { - if (use->GetUser() != nullptr && use->RequiresRegister()) { + // Process uses in the range (interval->GetStart(), interval->GetEnd()], i.e. + // [interval->GetStart() + 1, interval->GetEnd() + 1) + auto matching_use_range = FindMatchingUseRange(interval->GetUses().begin(), + interval->GetUses().end(), + interval->GetStart() + 1u, + interval->GetEnd() + 1u); + for (const UsePosition& use : matching_use_range) { + if (use.GetUser() != nullptr && use.RequiresRegister()) { // Cost for spilling at a register use point. - use_weight += CostForMoveAt(use->GetUser()->GetLifetimePosition() - 1, liveness); + use_weight += CostForMoveAt(use.GetUser()->GetLifetimePosition() - 1, liveness); } - use = use->GetNext(); } // We divide by the length of the interval because we want to prioritize @@ -989,16 +988,16 @@ void RegisterAllocatorGraphColor::SplitAtRegisterUses(LiveInterval* interval) { interval = TrySplit(interval, interval->GetStart() + 1); } - UsePosition* use = interval->GetFirstUse(); - while (use != nullptr && use->GetPosition() < interval->GetStart()) { - use = use->GetNext(); - } - + // Process uses in the range [interval->GetStart(), interval->GetEnd()], i.e. + // [interval->GetStart(), interval->GetEnd() + 1) + auto matching_use_range = FindMatchingUseRange(interval->GetUses().begin(), + interval->GetUses().end(), + interval->GetStart(), + interval->GetEnd() + 1u); // Split around register uses. - size_t end = interval->GetEnd(); - while (use != nullptr && use->GetPosition() <= end) { - if (use->RequiresRegister()) { - size_t position = use->GetPosition(); + for (const UsePosition& use : matching_use_range) { + if (use.RequiresRegister()) { + size_t position = use.GetPosition(); interval = TrySplit(interval, position - 1); if (liveness_.GetInstructionFromPosition(position / 2)->IsControlFlow()) { // If we are at the very end of a basic block, we cannot split right @@ -1008,7 +1007,6 @@ void RegisterAllocatorGraphColor::SplitAtRegisterUses(LiveInterval* interval) { interval = TrySplit(interval, position); } } - use = use->GetNext(); } } @@ -1398,18 +1396,20 @@ void ColoringIteration::FindCoalesceOpportunities() { } // Try to prevent moves into fixed input locations. - UsePosition* use = interval->GetFirstUse(); - for (; use != nullptr && use->GetPosition() <= interval->GetStart(); use = use->GetNext()) { - // Skip past uses before the start of this interval. - } - for (; use != nullptr && use->GetPosition() <= interval->GetEnd(); use = use->GetNext()) { - HInstruction* user = use->GetUser(); + // Process uses in the range (interval->GetStart(), interval->GetEnd()], i.e. + // [interval->GetStart() + 1, interval->GetEnd() + 1) + auto matching_use_range = FindMatchingUseRange(interval->GetUses().begin(), + interval->GetUses().end(), + interval->GetStart() + 1u, + interval->GetEnd() + 1u); + for (const UsePosition& use : matching_use_range) { + HInstruction* user = use.GetUser(); if (user == nullptr) { // User may be null for certain intervals, such as temp intervals. continue; } LocationSummary* locations = user->GetLocations(); - Location input = locations->InAt(use->GetInputIndex()); + Location input = locations->InAt(use.GetInputIndex()); if (input.IsRegister() || input.IsFpuRegister()) { // TODO: Could try to handle pair interval too, but coalescing with fixed pair nodes // is currently not supported. diff --git a/compiler/optimizing/register_allocator_test.cc b/compiler/optimizing/register_allocator_test.cc index 667afb1ec3..24a2ab24d8 100644 --- a/compiler/optimizing/register_allocator_test.cc +++ b/compiler/optimizing/register_allocator_test.cc @@ -912,9 +912,9 @@ TEST_F(RegisterAllocatorTest, SpillInactive) { // Create an interval with lifetime holes. static constexpr size_t ranges1[][2] = {{0, 2}, {4, 6}, {8, 10}}; LiveInterval* first = BuildInterval(ranges1, arraysize(ranges1), &allocator, -1, one); - first->first_use_ = new(&allocator) UsePosition(user, false, 8, first->first_use_); - first->first_use_ = new(&allocator) UsePosition(user, false, 7, first->first_use_); - first->first_use_ = new(&allocator) UsePosition(user, false, 6, first->first_use_); + first->uses_.push_front(*new(&allocator) UsePosition(user, false, 8)); + first->uses_.push_front(*new(&allocator) UsePosition(user, false, 7)); + first->uses_.push_front(*new(&allocator) UsePosition(user, false, 6)); locations = new (&allocator) LocationSummary(first->GetDefinedBy(), LocationSummary::kNoCall); locations->SetOut(Location::RequiresRegister()); @@ -934,9 +934,9 @@ TEST_F(RegisterAllocatorTest, SpillInactive) { // before lifetime position 6 yet. static constexpr size_t ranges3[][2] = {{2, 4}, {8, 10}}; LiveInterval* third = BuildInterval(ranges3, arraysize(ranges3), &allocator, -1, three); - third->first_use_ = new(&allocator) UsePosition(user, false, 8, third->first_use_); - third->first_use_ = new(&allocator) UsePosition(user, false, 4, third->first_use_); - third->first_use_ = new(&allocator) UsePosition(user, false, 3, third->first_use_); + third->uses_.push_front(*new(&allocator) UsePosition(user, false, 8)); + third->uses_.push_front(*new(&allocator) UsePosition(user, false, 4)); + third->uses_.push_front(*new(&allocator) UsePosition(user, false, 3)); locations = new (&allocator) LocationSummary(third->GetDefinedBy(), LocationSummary::kNoCall); locations->SetOut(Location::RequiresRegister()); third = third->SplitAt(3); diff --git a/compiler/optimizing/scheduler.cc b/compiler/optimizing/scheduler.cc index 320f01a727..3e373d16fb 100644 --- a/compiler/optimizing/scheduler.cc +++ b/compiler/optimizing/scheduler.cc @@ -66,28 +66,215 @@ static bool MayHaveReorderingDependency(SideEffects node, SideEffects other) { return false; } +size_t SchedulingGraph::ArrayAccessHeapLocation(HInstruction* array, HInstruction* index) const { + DCHECK(heap_location_collector_ != nullptr); + size_t heap_loc = heap_location_collector_->GetArrayAccessHeapLocation(array, index); + // This array access should be analyzed and added to HeapLocationCollector before. + DCHECK(heap_loc != HeapLocationCollector::kHeapLocationNotFound); + return heap_loc; +} -// Check whether `node` depends on `other`, taking into account `SideEffect` -// information and `CanThrow` information. -static bool HasSideEffectDependency(const HInstruction* node, const HInstruction* other) { - if (MayHaveReorderingDependency(node->GetSideEffects(), other->GetSideEffects())) { +bool SchedulingGraph::ArrayAccessMayAlias(const HInstruction* node, + const HInstruction* other) const { + DCHECK(heap_location_collector_ != nullptr); + size_t node_heap_loc = ArrayAccessHeapLocation(node->InputAt(0), node->InputAt(1)); + size_t other_heap_loc = ArrayAccessHeapLocation(other->InputAt(0), other->InputAt(1)); + + // For example: arr[0] and arr[0] + if (node_heap_loc == other_heap_loc) { return true; } - if (other->CanThrow() && node->GetSideEffects().DoesAnyWrite()) { + // For example: arr[0] and arr[i] + if (heap_location_collector_->MayAlias(node_heap_loc, other_heap_loc)) { return true; } - if (other->GetSideEffects().DoesAnyWrite() && node->CanThrow()) { + return false; +} + +static bool IsArrayAccess(const HInstruction* instruction) { + return instruction->IsArrayGet() || instruction->IsArraySet(); +} + +static bool IsInstanceFieldAccess(const HInstruction* instruction) { + return instruction->IsInstanceFieldGet() || + instruction->IsInstanceFieldSet() || + instruction->IsUnresolvedInstanceFieldGet() || + instruction->IsUnresolvedInstanceFieldSet(); +} + +static bool IsStaticFieldAccess(const HInstruction* instruction) { + return instruction->IsStaticFieldGet() || + instruction->IsStaticFieldSet() || + instruction->IsUnresolvedStaticFieldGet() || + instruction->IsUnresolvedStaticFieldSet(); +} + +static bool IsResolvedFieldAccess(const HInstruction* instruction) { + return instruction->IsInstanceFieldGet() || + instruction->IsInstanceFieldSet() || + instruction->IsStaticFieldGet() || + instruction->IsStaticFieldSet(); +} + +static bool IsUnresolvedFieldAccess(const HInstruction* instruction) { + return instruction->IsUnresolvedInstanceFieldGet() || + instruction->IsUnresolvedInstanceFieldSet() || + instruction->IsUnresolvedStaticFieldGet() || + instruction->IsUnresolvedStaticFieldSet(); +} + +static bool IsFieldAccess(const HInstruction* instruction) { + return IsResolvedFieldAccess(instruction) || IsUnresolvedFieldAccess(instruction); +} + +static const FieldInfo* GetFieldInfo(const HInstruction* instruction) { + if (instruction->IsInstanceFieldGet()) { + return &instruction->AsInstanceFieldGet()->GetFieldInfo(); + } else if (instruction->IsInstanceFieldSet()) { + return &instruction->AsInstanceFieldSet()->GetFieldInfo(); + } else if (instruction->IsStaticFieldGet()) { + return &instruction->AsStaticFieldGet()->GetFieldInfo(); + } else if (instruction->IsStaticFieldSet()) { + return &instruction->AsStaticFieldSet()->GetFieldInfo(); + } else { + LOG(FATAL) << "Unexpected field access type"; + UNREACHABLE(); + } +} + +size_t SchedulingGraph::FieldAccessHeapLocation(HInstruction* obj, const FieldInfo* field) const { + DCHECK(obj != nullptr); + DCHECK(field != nullptr); + DCHECK(heap_location_collector_ != nullptr); + + size_t heap_loc = heap_location_collector_->FindHeapLocationIndex( + heap_location_collector_->FindReferenceInfoOf( + heap_location_collector_->HuntForOriginalReference(obj)), + field->GetFieldOffset().SizeValue(), + nullptr, + field->GetDeclaringClassDefIndex()); + // This field access should be analyzed and added to HeapLocationCollector before. + DCHECK(heap_loc != HeapLocationCollector::kHeapLocationNotFound); + + return heap_loc; +} + +bool SchedulingGraph::FieldAccessMayAlias(const HInstruction* node, + const HInstruction* other) const { + DCHECK(heap_location_collector_ != nullptr); + + // Static and instance field accesses should not alias. + if ((IsInstanceFieldAccess(node) && IsStaticFieldAccess(other)) || + (IsStaticFieldAccess(node) && IsInstanceFieldAccess(other))) { + return false; + } + + // If either of the field accesses is unresolved. + if (IsUnresolvedFieldAccess(node) || IsUnresolvedFieldAccess(other)) { + // Conservatively treat these two accesses may alias. + return true; + } + + // If both fields accesses are resolved. + const FieldInfo* node_field = GetFieldInfo(node); + const FieldInfo* other_field = GetFieldInfo(other); + + size_t node_loc = FieldAccessHeapLocation(node->InputAt(0), node_field); + size_t other_loc = FieldAccessHeapLocation(other->InputAt(0), other_field); + + if (node_loc == other_loc) { return true; } + if (!heap_location_collector_->MayAlias(node_loc, other_loc)) { + return false; + } + + return true; +} + +bool SchedulingGraph::HasMemoryDependency(const HInstruction* node, + const HInstruction* other) const { + if (!MayHaveReorderingDependency(node->GetSideEffects(), other->GetSideEffects())) { + return false; + } + + if (heap_location_collector_ == nullptr || + heap_location_collector_->GetNumberOfHeapLocations() == 0) { + // Without HeapLocation information from load store analysis, + // we cannot do further disambiguation analysis on these two instructions. + // Just simply say that those two instructions have memory dependency. + return true; + } + + if (IsArrayAccess(node) && IsArrayAccess(other)) { + return ArrayAccessMayAlias(node, other); + } + if (IsFieldAccess(node) && IsFieldAccess(other)) { + return FieldAccessMayAlias(node, other); + } + + // TODO(xueliang): LSA to support alias analysis among HVecLoad, HVecStore and ArrayAccess + if (node->IsVecMemoryOperation() && other->IsVecMemoryOperation()) { + return true; + } + if (node->IsVecMemoryOperation() && IsArrayAccess(other)) { + return true; + } + if (IsArrayAccess(node) && other->IsVecMemoryOperation()) { + return true; + } + + // Heap accesses of different kinds should not alias. + if (IsArrayAccess(node) && IsFieldAccess(other)) { + return false; + } + if (IsFieldAccess(node) && IsArrayAccess(other)) { + return false; + } + if (node->IsVecMemoryOperation() && IsFieldAccess(other)) { + return false; + } + if (IsFieldAccess(node) && other->IsVecMemoryOperation()) { + return false; + } + + // We conservatively treat all other cases having dependency, + // for example, Invoke and ArrayGet. + return true; +} + +bool SchedulingGraph::HasExceptionDependency(const HInstruction* node, + const HInstruction* other) const { + if (other->CanThrow() && node->GetSideEffects().DoesAnyWrite()) { + return true; + } + if (other->GetSideEffects().DoesAnyWrite() && node->CanThrow()) { + return true; + } if (other->CanThrow() && node->CanThrow()) { return true; } - // Check side-effect dependency between ArrayGet and BoundsCheck. - if (node->IsArrayGet() && other->IsBoundsCheck() && node->InputAt(1) == other) { + // Above checks should cover all cases where we cannot reorder two + // instructions which may throw exception. + return false; +} + +// Check whether `node` depends on `other`, taking into account `SideEffect` +// information and `CanThrow` information. +bool SchedulingGraph::HasSideEffectDependency(const HInstruction* node, + const HInstruction* other) const { + if (HasMemoryDependency(node, other)) { + return true; + } + + // Even if above memory dependency check has passed, it is still necessary to + // check dependencies between instructions that can throw and instructions + // that write to memory. + if (HasExceptionDependency(node, other)) { return true; } @@ -109,6 +296,10 @@ void SchedulingGraph::AddDependencies(HInstruction* instruction, bool is_schedul // barrier depend on it. for (HInstruction* other = instruction->GetNext(); other != nullptr; other = other->GetNext()) { SchedulingNode* other_node = GetNode(other); + CHECK(other_node != nullptr) + << other->DebugName() + << " is in block " << other->GetBlock()->GetBlockId() + << ", and expected in block " << instruction->GetBlock()->GetBlockId(); bool other_is_barrier = other_node->IsSchedulingBarrier(); if (is_scheduling_barrier || other_is_barrier) { AddOtherDependency(other_node, instruction_node); @@ -363,6 +554,14 @@ SchedulingNode* CriticalPathSchedulingNodeSelector::GetHigherPrioritySchedulingN } void HScheduler::Schedule(HGraph* graph) { + // We run lsa here instead of in a separate pass to better control whether we + // should run the analysis or not. + LoadStoreAnalysis lsa(graph); + if (!only_optimize_loop_blocks_ || graph->HasLoops()) { + lsa.Run(); + scheduling_graph_.SetHeapLocationCollector(lsa.GetHeapLocationCollector()); + } + for (HBasicBlock* block : graph->GetReversePostOrder()) { if (IsSchedulable(block)) { Schedule(block); @@ -377,6 +576,10 @@ void HScheduler::Schedule(HBasicBlock* block) { scheduling_graph_.Clear(); for (HBackwardInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { HInstruction* instruction = it.Current(); + CHECK_EQ(instruction->GetBlock(), block) + << instruction->DebugName() + << " is in block " << instruction->GetBlock()->GetBlockId() + << ", and expected in block " << block->GetBlockId(); SchedulingNode* node = scheduling_graph_.AddNode(instruction, IsSchedulingBarrier(instruction)); CalculateLatency(node); scheduling_nodes.push_back(node); @@ -598,7 +801,9 @@ void HInstructionScheduling::Run(bool only_optimize_loop_blocks, // Avoid compilation error when compiling for unsupported instruction set. UNUSED(only_optimize_loop_blocks); UNUSED(schedule_randomly); + UNUSED(codegen_); #endif + switch (instruction_set_) { #ifdef ART_ENABLE_CODEGEN_arm64 case kArm64: { diff --git a/compiler/optimizing/scheduler.h b/compiler/optimizing/scheduler.h index 73e8087cd0..930a2c82cf 100644 --- a/compiler/optimizing/scheduler.h +++ b/compiler/optimizing/scheduler.h @@ -21,6 +21,7 @@ #include "base/time_utils.h" #include "driver/compiler_driver.h" +#include "load_store_analysis.h" #include "nodes.h" #include "optimization.h" #include "code_generator.h" @@ -246,7 +247,8 @@ class SchedulingGraph : public ValueObject { : scheduler_(scheduler), arena_(arena), contains_scheduling_barrier_(false), - nodes_map_(arena_->Adapter(kArenaAllocScheduler)) {} + nodes_map_(arena_->Adapter(kArenaAllocScheduler)), + heap_location_collector_(nullptr) {} SchedulingNode* AddNode(HInstruction* instr, bool is_scheduling_barrier = false) { SchedulingNode* node = new (arena_) SchedulingNode(instr, arena_, is_scheduling_barrier); @@ -261,6 +263,10 @@ class SchedulingGraph : public ValueObject { contains_scheduling_barrier_ = false; } + void SetHeapLocationCollector(const HeapLocationCollector& heap_location_collector) { + heap_location_collector_ = &heap_location_collector; + } + SchedulingNode* GetNode(const HInstruction* instr) const { auto it = nodes_map_.Find(instr); if (it == nodes_map_.end()) { @@ -294,6 +300,13 @@ class SchedulingGraph : public ValueObject { void AddOtherDependency(SchedulingNode* node, SchedulingNode* dependency) { AddDependency(node, dependency, /*is_data_dependency*/false); } + bool HasMemoryDependency(const HInstruction* node, const HInstruction* other) const; + bool HasExceptionDependency(const HInstruction* node, const HInstruction* other) const; + bool HasSideEffectDependency(const HInstruction* node, const HInstruction* other) const; + bool ArrayAccessMayAlias(const HInstruction* node, const HInstruction* other) const; + bool FieldAccessMayAlias(const HInstruction* node, const HInstruction* other) const; + size_t ArrayAccessHeapLocation(HInstruction* array, HInstruction* index) const; + size_t FieldAccessHeapLocation(HInstruction* obj, const FieldInfo* field) const; // Add dependencies nodes for the given `HInstruction`: inputs, environments, and side-effects. void AddDependencies(HInstruction* instruction, bool is_scheduling_barrier = false); @@ -305,6 +318,8 @@ class SchedulingGraph : public ValueObject { bool contains_scheduling_barrier_; ArenaHashMap<const HInstruction*, SchedulingNode*> nodes_map_; + + const HeapLocationCollector* heap_location_collector_; }; /* @@ -482,10 +497,9 @@ class HInstructionScheduling : public HOptimization { static constexpr const char* kInstructionScheduling = "scheduler"; + private: CodeGenerator* const codegen_; const InstructionSet instruction_set_; - - private: DISALLOW_COPY_AND_ASSIGN(HInstructionScheduling); }; diff --git a/compiler/optimizing/scheduler_arm.cc b/compiler/optimizing/scheduler_arm.cc index 832a7e1571..e78cd78aa2 100644 --- a/compiler/optimizing/scheduler_arm.cc +++ b/compiler/optimizing/scheduler_arm.cc @@ -818,10 +818,5 @@ void SchedulingLatencyVisitorARM::VisitTypeConversion(HTypeConversion* instr) { } } -void SchedulingLatencyVisitorARM::VisitArmDexCacheArraysBase(art::HArmDexCacheArraysBase*) { - last_visited_internal_latency_ = kArmIntegerOpLatency; - last_visited_latency_ = kArmIntegerOpLatency; -} - } // namespace arm } // namespace art diff --git a/compiler/optimizing/scheduler_arm.h b/compiler/optimizing/scheduler_arm.h index 897e97da49..a9f2295c35 100644 --- a/compiler/optimizing/scheduler_arm.h +++ b/compiler/optimizing/scheduler_arm.h @@ -17,20 +17,13 @@ #ifndef ART_COMPILER_OPTIMIZING_SCHEDULER_ARM_H_ #define ART_COMPILER_OPTIMIZING_SCHEDULER_ARM_H_ -#ifdef ART_USE_OLD_ARM_BACKEND -#include "code_generator_arm.h" -#else #include "code_generator_arm_vixl.h" -#endif #include "scheduler.h" namespace art { namespace arm { -#ifdef ART_USE_OLD_ARM_BACKEND -typedef CodeGeneratorARM CodeGeneratorARMType; -#else +// TODO: Replace CodeGeneratorARMType with CodeGeneratorARMVIXL everywhere? typedef CodeGeneratorARMVIXL CodeGeneratorARMType; -#endif // AArch32 instruction latencies. // We currently assume that all ARM CPUs share the same instruction latency list. diff --git a/compiler/optimizing/scheduler_test.cc b/compiler/optimizing/scheduler_test.cc index d87600aa5e..10c3cd7535 100644 --- a/compiler/optimizing/scheduler_test.cc +++ b/compiler/optimizing/scheduler_test.cc @@ -18,6 +18,7 @@ #include "builder.h" #include "codegen_test_utils.h" #include "common_compiler_test.h" +#include "load_store_analysis.h" #include "nodes.h" #include "optimizing_unit_test.h" #include "pc_relative_fixups_x86.h" @@ -40,8 +41,8 @@ static ::std::vector<CodegenTargetConfig> GetTargetConfigs() { ::std::vector<CodegenTargetConfig> v; ::std::vector<CodegenTargetConfig> test_config_candidates = { #ifdef ART_ENABLE_CODEGEN_arm - CodegenTargetConfig(kArm, create_codegen_arm), - CodegenTargetConfig(kThumb2, create_codegen_arm), + // TODO: Should't this be `kThumb2` instead of `kArm` here? + CodegenTargetConfig(kArm, create_codegen_arm_vixl32), #endif #ifdef ART_ENABLE_CODEGEN_arm64 CodegenTargetConfig(kArm64, create_codegen_arm64), @@ -193,6 +194,147 @@ class SchedulerTest : public CommonCompilerTest { } } + void TestDependencyGraphOnAliasingArrayAccesses(HScheduler* scheduler) { + HBasicBlock* entry = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(entry); + graph_->SetEntryBlock(entry); + graph_->BuildDominatorTree(); + + HInstruction* arr = new (&allocator_) HParameterValue(graph_->GetDexFile(), + dex::TypeIndex(0), + 0, + Primitive::kPrimNot); + HInstruction* i = new (&allocator_) HParameterValue(graph_->GetDexFile(), + dex::TypeIndex(1), + 1, + Primitive::kPrimInt); + HInstruction* j = new (&allocator_) HParameterValue(graph_->GetDexFile(), + dex::TypeIndex(1), + 1, + Primitive::kPrimInt); + HInstruction* object = new (&allocator_) HParameterValue(graph_->GetDexFile(), + dex::TypeIndex(0), + 0, + Primitive::kPrimNot); + HInstruction* c0 = graph_->GetIntConstant(0); + HInstruction* c1 = graph_->GetIntConstant(1); + HInstruction* add0 = new (&allocator_) HAdd(Primitive::kPrimInt, i, c0); + HInstruction* add1 = new (&allocator_) HAdd(Primitive::kPrimInt, i, c1); + HInstruction* sub0 = new (&allocator_) HSub(Primitive::kPrimInt, i, c0); + HInstruction* sub1 = new (&allocator_) HSub(Primitive::kPrimInt, i, c1); + HInstruction* arr_set_0 = new (&allocator_) HArraySet(arr, c0, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_1 = new (&allocator_) HArraySet(arr, c1, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_i = new (&allocator_) HArraySet(arr, i, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_add0 = new (&allocator_) HArraySet(arr, add0, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_add1 = new (&allocator_) HArraySet(arr, add1, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_sub0 = new (&allocator_) HArraySet(arr, sub0, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_sub1 = new (&allocator_) HArraySet(arr, sub1, c0, Primitive::kPrimInt, 0); + HInstruction* arr_set_j = new (&allocator_) HArraySet(arr, j, c0, Primitive::kPrimInt, 0); + HInstanceFieldSet* set_field10 = new (&allocator_) HInstanceFieldSet(object, + c1, + nullptr, + Primitive::kPrimInt, + MemberOffset(10), + false, + kUnknownFieldIndex, + kUnknownClassDefIndex, + graph_->GetDexFile(), + 0); + + HInstruction* block_instructions[] = {arr, + i, + j, + object, + add0, + add1, + sub0, + sub1, + arr_set_0, + arr_set_1, + arr_set_i, + arr_set_add0, + arr_set_add1, + arr_set_sub0, + arr_set_sub1, + arr_set_j, + set_field10}; + + for (HInstruction* instr : block_instructions) { + entry->AddInstruction(instr); + } + + SchedulingGraph scheduling_graph(scheduler, graph_->GetArena()); + HeapLocationCollector heap_location_collector(graph_); + heap_location_collector.VisitBasicBlock(entry); + heap_location_collector.BuildAliasingMatrix(); + scheduling_graph.SetHeapLocationCollector(heap_location_collector); + + for (HInstruction* instr : ReverseRange(block_instructions)) { + // Build scheduling graph with memory access aliasing information + // from LSA/heap_location_collector. + scheduling_graph.AddNode(instr); + } + + // LSA/HeapLocationCollector should see those ArraySet instructions. + ASSERT_EQ(heap_location_collector.GetNumberOfHeapLocations(), 9U); + ASSERT_TRUE(heap_location_collector.HasHeapStores()); + + // Test queries on HeapLocationCollector's aliasing matrix after load store analysis. + // HeapLocationCollector and SchedulingGraph should report consistent relationships. + size_t loc1 = HeapLocationCollector::kHeapLocationNotFound; + size_t loc2 = HeapLocationCollector::kHeapLocationNotFound; + + // Test side effect dependency: array[0] and array[1] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(arr, c0); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(arr, c1); + ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); + ASSERT_FALSE(scheduling_graph.HasImmediateOtherDependency(arr_set_1, arr_set_0)); + + // Test side effect dependency based on LSA analysis: array[i] and array[j] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(arr, i); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(arr, j); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_j, arr_set_i)); + + // Test side effect dependency based on LSA analysis: array[i] and array[i+0] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(arr, i); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(arr, add0); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_add0, arr_set_i)); + + // Test side effect dependency based on LSA analysis: array[i] and array[i-0] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(arr, i); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(arr, sub0); + ASSERT_TRUE(heap_location_collector.MayAlias(loc1, loc2)); + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_sub0, arr_set_i)); + + // Test side effect dependency based on LSA analysis: array[i] and array[i+1] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(arr, i); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(arr, add1); + ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); + ASSERT_FALSE(scheduling_graph.HasImmediateOtherDependency(arr_set_add1, arr_set_i)); + + // Test side effect dependency based on LSA analysis: array[i+1] and array[i-1] + loc1 = heap_location_collector.GetArrayAccessHeapLocation(arr, add1); + loc2 = heap_location_collector.GetArrayAccessHeapLocation(arr, sub1); + ASSERT_FALSE(heap_location_collector.MayAlias(loc1, loc2)); + ASSERT_FALSE(scheduling_graph.HasImmediateOtherDependency(arr_set_sub1, arr_set_add1)); + + // Test side effect dependency based on LSA analysis: array[j] and all others array accesses + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_j, arr_set_i)); + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_j, arr_set_add0)); + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_j, arr_set_sub0)); + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_j, arr_set_add1)); + ASSERT_TRUE(scheduling_graph.HasImmediateOtherDependency(arr_set_j, arr_set_sub1)); + + // Test that ArraySet and FieldSet should not have side effect dependency + ASSERT_FALSE(scheduling_graph.HasImmediateOtherDependency(arr_set_i, set_field10)); + ASSERT_FALSE(scheduling_graph.HasImmediateOtherDependency(arr_set_j, set_field10)); + + // Exercise target specific scheduler and SchedulingLatencyVisitor. + scheduler->Schedule(graph_); + } + ArenaPool pool_; ArenaAllocator allocator_; HGraph* graph_; @@ -204,15 +346,28 @@ TEST_F(SchedulerTest, DependencyGraphAndSchedulerARM64) { arm64::HSchedulerARM64 scheduler(&allocator_, &critical_path_selector); TestBuildDependencyGraphAndSchedule(&scheduler); } + +TEST_F(SchedulerTest, ArrayAccessAliasingARM64) { + CriticalPathSchedulingNodeSelector critical_path_selector; + arm64::HSchedulerARM64 scheduler(&allocator_, &critical_path_selector); + TestDependencyGraphOnAliasingArrayAccesses(&scheduler); +} #endif #if defined(ART_ENABLE_CODEGEN_arm) -TEST_F(SchedulerTest, DependencyGrapAndSchedulerARM) { +TEST_F(SchedulerTest, DependencyGraphAndSchedulerARM) { CriticalPathSchedulingNodeSelector critical_path_selector; arm::SchedulingLatencyVisitorARM arm_latency_visitor(/*CodeGenerator*/ nullptr); arm::HSchedulerARM scheduler(&allocator_, &critical_path_selector, &arm_latency_visitor); TestBuildDependencyGraphAndSchedule(&scheduler); } + +TEST_F(SchedulerTest, ArrayAccessAliasingARM) { + CriticalPathSchedulingNodeSelector critical_path_selector; + arm::SchedulingLatencyVisitorARM arm_latency_visitor(/*CodeGenerator*/ nullptr); + arm::HSchedulerARM scheduler(&allocator_, &critical_path_selector, &arm_latency_visitor); + TestDependencyGraphOnAliasingArrayAccesses(&scheduler); +} #endif TEST_F(SchedulerTest, RandomScheduling) { diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index 106b709eda..8bd568befd 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -128,15 +128,8 @@ void HSharpening::SharpenInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke, method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } else { - // Use PC-relative access to the dex cache arrays. - method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative; - // Note: we use the invoke's graph instead of the codegen graph, which are - // different when inlining (the codegen graph is the most outer graph). The - // invoke's dex method index is relative to the dex file where the invoke's graph - // was built from. - DexCacheArraysLayout layout(GetInstructionSetPointerSize(codegen->GetInstructionSet()), - &invoke->GetBlock()->GetGraph()->GetDexFile()); - method_load_data = layout.MethodOffset(invoke->GetDexMethodIndex()); + // Use PC-relative access to the .bss methods arrays. + method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kBssEntry; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } diff --git a/compiler/optimizing/ssa_liveness_analysis.cc b/compiler/optimizing/ssa_liveness_analysis.cc index b538a89a06..185303bc8c 100644 --- a/compiler/optimizing/ssa_liveness_analysis.cc +++ b/compiler/optimizing/ssa_liveness_analysis.cc @@ -197,7 +197,7 @@ void SsaLivenessAnalysis::ComputeLiveRanges() { HInstruction* instruction = environment->GetInstructionAt(i); bool should_be_live = ShouldBeLiveForEnvironment(current, instruction); if (should_be_live) { - DCHECK(instruction->HasSsaIndex()); + CHECK(instruction->HasSsaIndex()) << instruction->DebugName(); live_in->SetBit(instruction->GetSsaIndex()); } if (instruction != nullptr) { @@ -356,14 +356,16 @@ int LiveInterval::FindFirstRegisterHint(size_t* free_until, } } - UsePosition* use = first_use_; size_t start = GetStart(); size_t end = GetEnd(); - while (use != nullptr && use->GetPosition() <= end) { - size_t use_position = use->GetPosition(); - if (use_position >= start && !use->IsSynthesized()) { - HInstruction* user = use->GetUser(); - size_t input_index = use->GetInputIndex(); + for (const UsePosition& use : GetUses()) { + size_t use_position = use.GetPosition(); + if (use_position > end) { + break; + } + if (use_position >= start && !use.IsSynthesized()) { + HInstruction* user = use.GetUser(); + size_t input_index = use.GetInputIndex(); if (user->IsPhi()) { // If the phi has a register, try to use the same. Location phi_location = user->GetLiveInterval()->ToLocation(); @@ -395,7 +397,7 @@ int LiveInterval::FindFirstRegisterHint(size_t* free_until, } else { // If the instruction is expected in a register, try to use it. LocationSummary* locations = user->GetLocations(); - Location expected = locations->InAt(use->GetInputIndex()); + Location expected = locations->InAt(use.GetInputIndex()); // We use the user's lifetime position - 1 (and not `use_position`) because the // register is blocked at the beginning of the user. size_t position = user->GetLifetimePosition() - 1; @@ -408,7 +410,6 @@ int LiveInterval::FindFirstRegisterHint(size_t* free_until, } } } - use = use->GetNext(); } return kNoRegister; diff --git a/compiler/optimizing/ssa_liveness_analysis.h b/compiler/optimizing/ssa_liveness_analysis.h index e9dffc1fac..a6681575a2 100644 --- a/compiler/optimizing/ssa_liveness_analysis.h +++ b/compiler/optimizing/ssa_liveness_analysis.h @@ -19,7 +19,9 @@ #include <iostream> +#include "base/iteration_range.h" #include "nodes.h" +#include "utils/intrusive_forward_list.h" namespace art { @@ -102,28 +104,23 @@ class LiveRange FINAL : public ArenaObject<kArenaAllocSsaLiveness> { /** * A use position represents a live interval use at a given position. */ -class UsePosition : public ArenaObject<kArenaAllocSsaLiveness> { +class UsePosition : public ArenaObject<kArenaAllocSsaLiveness>, + public IntrusiveForwardListNode<UsePosition> { public: - UsePosition(HInstruction* user, size_t input_index, size_t position, UsePosition* next) + UsePosition(HInstruction* user, size_t input_index, size_t position) : user_(user), input_index_(input_index), - position_(position), - next_(next) { - DCHECK(next_ == nullptr || next->GetPosition() >= GetPosition()); + position_(position) { } explicit UsePosition(size_t position) : user_(nullptr), input_index_(kNoInput), - position_(dchecked_integral_cast<uint32_t>(position)), - next_(nullptr) { + position_(dchecked_integral_cast<uint32_t>(position)) { } size_t GetPosition() const { return position_; } - UsePosition* GetNext() const { return next_; } - void SetNext(UsePosition* next) { next_ = next; } - HInstruction* GetUser() const { return user_; } bool IsSynthesized() const { return user_ == nullptr; } @@ -138,10 +135,8 @@ class UsePosition : public ArenaObject<kArenaAllocSsaLiveness> { return user_->GetBlock()->GetLoopInformation(); } - UsePosition* Dup(ArenaAllocator* allocator) const { - return new (allocator) UsePosition( - user_, input_index_, position_, - next_ == nullptr ? nullptr : next_->Dup(allocator)); + UsePosition* Clone(ArenaAllocator* allocator) const { + return new (allocator) UsePosition(user_, input_index_, position_); } bool RequiresRegister() const { @@ -156,33 +151,28 @@ class UsePosition : public ArenaObject<kArenaAllocSsaLiveness> { HInstruction* const user_; const size_t input_index_; const size_t position_; - UsePosition* next_; DISALLOW_COPY_AND_ASSIGN(UsePosition); }; +using UsePositionList = IntrusiveForwardList<UsePosition>; /** * An environment use position represents a live interval for environment use at a given position. */ -class EnvUsePosition : public ArenaObject<kArenaAllocSsaLiveness> { +class EnvUsePosition : public ArenaObject<kArenaAllocSsaLiveness>, + public IntrusiveForwardListNode<EnvUsePosition> { public: EnvUsePosition(HEnvironment* environment, size_t input_index, - size_t position, - EnvUsePosition* next) + size_t position) : environment_(environment), input_index_(input_index), - position_(position), - next_(next) { + position_(position) { DCHECK(environment != nullptr); - DCHECK(next_ == nullptr || next->GetPosition() >= GetPosition()); } size_t GetPosition() const { return position_; } - EnvUsePosition* GetNext() const { return next_; } - void SetNext(EnvUsePosition* next) { next_ = next; } - HEnvironment* GetEnvironment() const { return environment_; } size_t GetInputIndex() const { return input_index_; } @@ -190,20 +180,47 @@ class EnvUsePosition : public ArenaObject<kArenaAllocSsaLiveness> { stream << position_; } - EnvUsePosition* Dup(ArenaAllocator* allocator) const { - return new (allocator) EnvUsePosition( - environment_, input_index_, position_, - next_ == nullptr ? nullptr : next_->Dup(allocator)); + EnvUsePosition* Clone(ArenaAllocator* allocator) const { + return new (allocator) EnvUsePosition(environment_, input_index_, position_); } private: HEnvironment* const environment_; const size_t input_index_; const size_t position_; - EnvUsePosition* next_; DISALLOW_COPY_AND_ASSIGN(EnvUsePosition); }; +using EnvUsePositionList = IntrusiveForwardList<EnvUsePosition>; + +template <typename Iterator> +inline Iterator FindUseAtOrAfterPosition(Iterator first, Iterator last, size_t position) { + using value_type = const typename Iterator::value_type; + static_assert(std::is_same<value_type, const UsePosition>::value || + std::is_same<value_type, const EnvUsePosition>::value, + "Expecting value type UsePosition or EnvUsePosition."); + Iterator ret = std::find_if( + first, last, [position](const value_type& use) { return use.GetPosition() >= position; }); + // Check that the processed range is sorted. Do not check the rest of the range to avoid + // increasing the complexity of callers from O(n) to O(n^2). + DCHECK(std::is_sorted( + first, + ret, + [](const value_type& lhs, const value_type& rhs) { + return lhs.GetPosition() < rhs.GetPosition(); + })); + return ret; +} + +template <typename Iterator> +inline IterationRange<Iterator> FindMatchingUseRange(Iterator first, + Iterator last, + size_t position_begin, + size_t position_end) { + Iterator begin = FindUseAtOrAfterPosition(first, last, position_begin); + Iterator end = FindUseAtOrAfterPosition(begin, last, position_end); + return MakeIterationRange(begin, end); +} class SafepointPosition : public ArenaObject<kArenaAllocSsaLiveness> { public: @@ -265,11 +282,11 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { void AddTempUse(HInstruction* instruction, size_t temp_index) { DCHECK(IsTemp()); - DCHECK(first_use_ == nullptr) << "A temporary can only have one user"; - DCHECK(first_env_use_ == nullptr) << "A temporary cannot have environment user"; + DCHECK(GetUses().empty()) << "A temporary can only have one user"; + DCHECK(GetEnvironmentUses().empty()) << "A temporary cannot have environment user"; size_t position = instruction->GetLifetimePosition(); - first_use_ = new (allocator_) UsePosition( - instruction, temp_index, position, first_use_); + UsePosition* new_use = new (allocator_) UsePosition(instruction, temp_index, position); + uses_.push_front(*new_use); AddRange(position, position + 1); } @@ -306,32 +323,36 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { AddBackEdgeUses(*instruction->GetBlock()); } - if ((first_use_ != nullptr) - && (first_use_->GetUser() == actual_user) - && (first_use_->GetPosition() < position)) { + if ((!uses_.empty()) && + (uses_.front().GetUser() == actual_user) && + (uses_.front().GetPosition() < position)) { // The user uses the instruction multiple times, and one use dies before the other. // We update the use list so that the latter is first. DCHECK(!is_environment); - UsePosition* cursor = first_use_; - while ((cursor->GetNext() != nullptr) && (cursor->GetNext()->GetPosition() < position)) { - cursor = cursor->GetNext(); - } - DCHECK(first_use_->GetPosition() + 1 == position); - UsePosition* new_use = new (allocator_) UsePosition( - instruction, input_index, position, cursor->GetNext()); - cursor->SetNext(new_use); - if (first_range_->GetEnd() == first_use_->GetPosition()) { + DCHECK(uses_.front().GetPosition() + 1 == position); + UsePositionList::iterator next_pos = uses_.begin(); + UsePositionList::iterator insert_pos; + do { + insert_pos = next_pos; + ++next_pos; + } while (next_pos != uses_.end() && next_pos->GetPosition() < position); + UsePosition* new_use = new (allocator_) UsePosition(instruction, input_index, position); + uses_.insert_after(insert_pos, *new_use); + if (first_range_->GetEnd() == uses_.front().GetPosition()) { first_range_->end_ = position; } return; } if (is_environment) { - first_env_use_ = new (allocator_) EnvUsePosition( - environment, input_index, position, first_env_use_); + DCHECK(env_uses_.empty() || position <= env_uses_.front().GetPosition()); + EnvUsePosition* new_env_use = + new (allocator_) EnvUsePosition(environment, input_index, position); + env_uses_.push_front(*new_env_use); } else { - first_use_ = new (allocator_) UsePosition( - instruction, input_index, position, first_use_); + DCHECK(uses_.empty() || position <= uses_.front().GetPosition()); + UsePosition* new_use = new (allocator_) UsePosition(instruction, input_index, position); + uses_.push_front(*new_use); } if (is_environment && !keep_alive) { @@ -369,8 +390,9 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { if (block->IsInLoop()) { AddBackEdgeUses(*block); } - first_use_ = new (allocator_) UsePosition( - instruction, input_index, block->GetLifetimeEnd(), first_use_); + UsePosition* new_use = + new (allocator_) UsePosition(instruction, input_index, block->GetLifetimeEnd()); + uses_.push_front(*new_use); } ALWAYS_INLINE void AddRange(size_t start, size_t end) { @@ -430,7 +452,7 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { first_range_->start_ = from; } else { // Instruction without uses. - DCHECK(first_use_ == nullptr); + DCHECK(uses_.empty()); DCHECK(from == defined_by_->GetLifetimePosition()); first_range_ = last_range_ = range_search_start_ = new (allocator_) LiveRange(from, from + 2, nullptr); @@ -528,16 +550,17 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { return position; } - UsePosition* use = first_use_; size_t end = GetEnd(); - while (use != nullptr && use->GetPosition() <= end) { - size_t use_position = use->GetPosition(); + for (const UsePosition& use : GetUses()) { + size_t use_position = use.GetPosition(); + if (use_position > end) { + break; + } if (use_position > position) { - if (use->RequiresRegister()) { + if (use.RequiresRegister()) { return use_position; } } - use = use->GetNext(); } return kNoLifetime; } @@ -564,24 +587,25 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { return position; } - UsePosition* use = first_use_; size_t end = GetEnd(); - while (use != nullptr && use->GetPosition() <= end) { - size_t use_position = use->GetPosition(); + for (const UsePosition& use : GetUses()) { + size_t use_position = use.GetPosition(); + if (use_position > end) { + break; + } if (use_position > position) { return use_position; } - use = use->GetNext(); } return kNoLifetime; } - UsePosition* GetFirstUse() const { - return first_use_; + const UsePositionList& GetUses() const { + return parent_->uses_; } - EnvUsePosition* GetFirstEnvironmentUse() const { - return first_env_use_; + const EnvUsePositionList& GetEnvironmentUses() const { + return parent_->env_uses_; } Primitive::Type GetType() const { @@ -645,8 +669,6 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { next_sibling_ = new_interval; new_interval->parent_ = parent_; - new_interval->first_use_ = first_use_; - new_interval->first_env_use_ = first_env_use_; LiveRange* current = first_range_; LiveRange* previous = nullptr; // Iterate over the ranges, and either find a range that covers this position, or @@ -718,20 +740,14 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { current = current->GetNext(); } stream << "}, uses: { "; - const UsePosition* use = first_use_; - if (use != nullptr) { - do { - use->Dump(stream); - stream << " "; - } while ((use = use->GetNext()) != nullptr); + for (const UsePosition& use : GetUses()) { + use.Dump(stream); + stream << " "; } stream << "}, { "; - const EnvUsePosition* env_use = first_env_use_; - if (env_use != nullptr) { - do { - env_use->Dump(stream); - stream << " "; - } while ((env_use = env_use->GetNext()) != nullptr); + for (const EnvUsePosition& env_use : GetEnvironmentUses()) { + env_use.Dump(stream); + stream << " "; } stream << "}"; stream << " is_fixed: " << is_fixed_ << ", is_split: " << IsSplit(); @@ -833,12 +849,16 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { high_or_low_interval_->last_range_ = high_or_low_interval_->first_range_->GetLastRange(); high_or_low_interval_->range_search_start_ = high_or_low_interval_->first_range_; } - if (first_use_ != nullptr) { - high_or_low_interval_->first_use_ = first_use_->Dup(allocator_); + auto pos = high_or_low_interval_->uses_.before_begin(); + for (const UsePosition& use : uses_) { + UsePosition* new_use = use.Clone(allocator_); + pos = high_or_low_interval_->uses_.insert_after(pos, *new_use); } - if (first_env_use_ != nullptr) { - high_or_low_interval_->first_env_use_ = first_env_use_->Dup(allocator_); + auto env_pos = high_or_low_interval_->env_uses_.before_begin(); + for (const EnvUsePosition& env_use : env_uses_) { + EnvUsePosition* new_env_use = env_use.Clone(allocator_); + env_pos = high_or_low_interval_->env_uses_.insert_after(env_pos, *new_env_use); } } @@ -962,8 +982,8 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { range_search_start_(nullptr), first_safepoint_(nullptr), last_safepoint_(nullptr), - first_use_(nullptr), - first_env_use_(nullptr), + uses_(), + env_uses_(), type_(type), next_sibling_(nullptr), parent_(this), @@ -1005,14 +1025,12 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { } bool HasSynthesizeUseAt(size_t position) const { - UsePosition* use = first_use_; - while (use != nullptr) { - size_t use_position = use->GetPosition(); - if ((use_position == position) && use->IsSynthesized()) { + for (const UsePosition& use : GetUses()) { + size_t use_position = use.GetPosition(); + if ((use_position == position) && use.IsSynthesized()) { return true; } if (use_position > position) break; - use = use->GetNext(); } return false; } @@ -1028,11 +1046,11 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { // Add synthesized uses at the back edge of loops to help the register allocator. // Note that this method is called in decreasing liveness order, to faciliate adding - // uses at the head of the `first_use_` linked list. Because below + // uses at the head of the `uses_` list. Because below // we iterate from inner-most to outer-most, which is in increasing liveness order, - // we need to take extra care of how the `first_use_` linked list is being updated. - UsePosition* first_in_new_list = nullptr; - UsePosition* last_in_new_list = nullptr; + // we need to add subsequent entries after the last inserted entry. + const UsePositionList::iterator old_begin = uses_.begin(); + UsePositionList::iterator insert_pos = uses_.before_begin(); for (HLoopInformationOutwardIterator it(block_at_use); !it.Done(); it.Advance()) { @@ -1042,37 +1060,25 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { break; } - // We're only adding a synthesized use at the last back edge. Adding syntehsized uses on + // We're only adding a synthesized use at the last back edge. Adding synthesized uses on // all back edges is not necessary: anything used in the loop will have its use at the // last back edge. If we want branches in a loop to have better register allocation than // another branch, then it is the linear order we should change. size_t back_edge_use_position = current->GetLifetimeEnd(); - if ((first_use_ != nullptr) && (first_use_->GetPosition() <= back_edge_use_position)) { + if ((old_begin != uses_.end()) && (old_begin->GetPosition() <= back_edge_use_position)) { // There was a use already seen in this loop. Therefore the previous call to `AddUse` // already inserted the backedge use. We can stop going outward. DCHECK(HasSynthesizeUseAt(back_edge_use_position)); break; } - DCHECK(last_in_new_list == nullptr || - back_edge_use_position > last_in_new_list->GetPosition()); + DCHECK(insert_pos != uses_.before_begin() + ? back_edge_use_position > insert_pos->GetPosition() + : current == block_at_use.GetLoopInformation()) + << std::distance(uses_.before_begin(), insert_pos); UsePosition* new_use = new (allocator_) UsePosition(back_edge_use_position); - - if (last_in_new_list != nullptr) { - // Going outward. The latest created use needs to point to the new use. - last_in_new_list->SetNext(new_use); - } else { - // This is the inner-most loop. - DCHECK_EQ(current, block_at_use.GetLoopInformation()); - first_in_new_list = new_use; - } - last_in_new_list = new_use; - } - // Link the newly created linked list with `first_use_`. - if (last_in_new_list != nullptr) { - last_in_new_list->SetNext(first_use_); - first_use_ = first_in_new_list; + insert_pos = uses_.insert_after(insert_pos, *new_use); } } @@ -1091,9 +1097,9 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { SafepointPosition* first_safepoint_; SafepointPosition* last_safepoint_; - // Uses of this interval. Note that this linked list is shared amongst siblings. - UsePosition* first_use_; - EnvUsePosition* first_env_use_; + // Uses of this interval. Only the parent interval keeps these lists. + UsePositionList uses_; + EnvUsePositionList env_uses_; // The instruction type this interval corresponds to. const Primitive::Type type_; @@ -1202,14 +1208,14 @@ class SsaLivenessAnalysis : public ValueObject { // A temporary shares the same lifetime start as the instruction that requires it. DCHECK(temp->IsTemp()); HInstruction* user = GetInstructionFromPosition(temp->GetStart() / 2); - DCHECK_EQ(user, temp->GetFirstUse()->GetUser()); + DCHECK_EQ(user, temp->GetUses().front().GetUser()); return user; } size_t GetTempIndex(LiveInterval* temp) const { // We use the input index to store the index of the temporary in the user's temporary list. DCHECK(temp->IsTemp()); - return temp->GetFirstUse()->GetInputIndex(); + return temp->GetUses().front().GetInputIndex(); } size_t GetMaxLifetimePosition() const { diff --git a/compiler/utils/arm/assembler_arm.cc b/compiler/utils/arm/assembler_arm.cc deleted file mode 100644 index d5cd59d481..0000000000 --- a/compiler/utils/arm/assembler_arm.cc +++ /dev/null @@ -1,453 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "assembler_arm.h" - -#include <algorithm> - -#include "base/bit_utils.h" -#include "base/logging.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "offsets.h" -#include "thread.h" - -namespace art { -namespace arm { - -const char* kRegisterNames[] = { - "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", - "fp", "ip", "sp", "lr", "pc" -}; - -const char* kConditionNames[] = { - "EQ", "NE", "CS", "CC", "MI", "PL", "VS", "VC", "HI", "LS", "GE", "LT", "GT", - "LE", "AL", -}; - -std::ostream& operator<<(std::ostream& os, const Register& rhs) { - if (rhs >= R0 && rhs <= PC) { - os << kRegisterNames[rhs]; - } else { - os << "Register[" << static_cast<int>(rhs) << "]"; - } - return os; -} - - -std::ostream& operator<<(std::ostream& os, const SRegister& rhs) { - if (rhs >= S0 && rhs < kNumberOfSRegisters) { - os << "s" << static_cast<int>(rhs); - } else { - os << "SRegister[" << static_cast<int>(rhs) << "]"; - } - return os; -} - - -std::ostream& operator<<(std::ostream& os, const DRegister& rhs) { - if (rhs >= D0 && rhs < kNumberOfDRegisters) { - os << "d" << static_cast<int>(rhs); - } else { - os << "DRegister[" << static_cast<int>(rhs) << "]"; - } - return os; -} - -std::ostream& operator<<(std::ostream& os, const Condition& rhs) { - if (rhs >= EQ && rhs <= AL) { - os << kConditionNames[rhs]; - } else { - os << "Condition[" << static_cast<int>(rhs) << "]"; - } - return os; -} - -ShifterOperand::ShifterOperand(uint32_t immed) - : type_(kImmediate), rm_(kNoRegister), rs_(kNoRegister), - is_rotate_(false), is_shift_(false), shift_(kNoShift), rotate_(0), immed_(immed) { - CHECK(immed < (1u << 12) || ArmAssembler::ModifiedImmediate(immed) != kInvalidModifiedImmediate); -} - - -uint32_t ShifterOperand::encodingArm() const { - CHECK(is_valid()); - switch (type_) { - case kImmediate: - if (is_rotate_) { - return (rotate_ << kRotateShift) | (immed_ << kImmed8Shift); - } else { - return immed_; - } - case kRegister: - if (is_shift_) { - uint32_t shift_type; - switch (shift_) { - case arm::Shift::ROR: - shift_type = static_cast<uint32_t>(shift_); - CHECK_NE(immed_, 0U); - break; - case arm::Shift::RRX: - shift_type = static_cast<uint32_t>(arm::Shift::ROR); // Same encoding as ROR. - CHECK_EQ(immed_, 0U); - break; - default: - shift_type = static_cast<uint32_t>(shift_); - } - // Shifted immediate or register. - if (rs_ == kNoRegister) { - // Immediate shift. - return immed_ << kShiftImmShift | - shift_type << kShiftShift | - static_cast<uint32_t>(rm_); - } else { - // Register shift. - return static_cast<uint32_t>(rs_) << kShiftRegisterShift | - shift_type << kShiftShift | (1 << 4) | - static_cast<uint32_t>(rm_); - } - } else { - // Simple register - return static_cast<uint32_t>(rm_); - } - default: - // Can't get here. - LOG(FATAL) << "Invalid shifter operand for ARM"; - return 0; - } -} - -uint32_t ShifterOperand::encodingThumb() const { - switch (type_) { - case kImmediate: - return immed_; - case kRegister: - if (is_shift_) { - // Shifted immediate or register. - if (rs_ == kNoRegister) { - // Immediate shift. - if (shift_ == RRX) { - DCHECK_EQ(immed_, 0u); - // RRX is encoded as an ROR with imm 0. - return ROR << 4 | static_cast<uint32_t>(rm_); - } else { - DCHECK((1 <= immed_ && immed_ <= 31) || - (immed_ == 0u && shift_ == LSL) || - (immed_ == 32u && (shift_ == ASR || shift_ == LSR))); - uint32_t imm3 = (immed_ >> 2) & 7 /* 0b111*/; - uint32_t imm2 = immed_ & 3U /* 0b11 */; - - return imm3 << 12 | imm2 << 6 | shift_ << 4 | - static_cast<uint32_t>(rm_); - } - } else { - LOG(FATAL) << "No register-shifted register instruction available in thumb"; - return 0; - } - } else { - // Simple register - return static_cast<uint32_t>(rm_); - } - default: - // Can't get here. - LOG(FATAL) << "Invalid shifter operand for thumb"; - UNREACHABLE(); - } -} - -uint32_t Address::encodingArm() const { - CHECK(IsAbsoluteUint<12>(offset_)); - uint32_t encoding; - if (is_immed_offset_) { - if (offset_ < 0) { - encoding = (am_ ^ (1 << kUShift)) | -offset_; // Flip U to adjust sign. - } else { - encoding = am_ | offset_; - } - } else { - uint32_t shift = shift_; - if (shift == RRX) { - CHECK_EQ(offset_, 0); - shift = ROR; - } - encoding = am_ | static_cast<uint32_t>(rm_) | shift << 5 | offset_ << 7 | B25; - } - encoding |= static_cast<uint32_t>(rn_) << kRnShift; - return encoding; -} - - -uint32_t Address::encodingThumb(bool is_32bit) const { - uint32_t encoding = 0; - if (is_immed_offset_) { - encoding = static_cast<uint32_t>(rn_) << 16; - // Check for the T3/T4 encoding. - // PUW must Offset for T3 - // Convert ARM PU0W to PUW - // The Mode is in ARM encoding format which is: - // |P|U|0|W| - // we need this in thumb2 mode: - // |P|U|W| - - uint32_t am = am_; - int32_t offset = offset_; - if (offset < 0) { - am ^= 1 << kUShift; - offset = -offset; - } - if (offset_ < 0 || (offset >= 0 && offset < 256 && - am_ != Mode::Offset)) { - // T4 encoding. - uint32_t PUW = am >> 21; // Move down to bottom of word. - PUW = (PUW >> 1) | (PUW & 1); // Bits 3, 2 and 0. - // If P is 0 then W must be 1 (Different from ARM). - if ((PUW & 4U /* 0b100 */) == 0) { - PUW |= 1U /* 0b1 */; - } - encoding |= B11 | PUW << 8 | offset; - } else { - // T3 encoding (also sets op1 to 0b01). - encoding |= B23 | offset_; - } - } else { - // Register offset, possibly shifted. - // Need to choose between encoding T1 (16 bit) or T2. - // Only Offset mode is supported. Shift must be LSL and the count - // is only 2 bits. - CHECK_EQ(shift_, LSL); - CHECK_LE(offset_, 4); - CHECK_EQ(am_, Offset); - bool is_t2 = is_32bit; - if (ArmAssembler::IsHighRegister(rn_) || ArmAssembler::IsHighRegister(rm_)) { - is_t2 = true; - } else if (offset_ != 0) { - is_t2 = true; - } - if (is_t2) { - encoding = static_cast<uint32_t>(rn_) << 16 | static_cast<uint32_t>(rm_) | - offset_ << 4; - } else { - encoding = static_cast<uint32_t>(rn_) << 3 | static_cast<uint32_t>(rm_) << 6; - } - } - return encoding; -} - -// This is very like the ARM encoding except the offset is 10 bits. -uint32_t Address::encodingThumbLdrdStrd() const { - DCHECK(IsImmediate()); - uint32_t encoding; - uint32_t am = am_; - // If P is 0 then W must be 1 (Different from ARM). - uint32_t PU1W = am_ >> 21; // Move down to bottom of word. - if ((PU1W & 8U /* 0b1000 */) == 0) { - am |= 1 << 21; // Set W bit. - } - if (offset_ < 0) { - int32_t off = -offset_; - CHECK_LT(off, 1024); - CHECK_ALIGNED(off, 4); - encoding = (am ^ (1 << kUShift)) | off >> 2; // Flip U to adjust sign. - } else { - CHECK_LT(offset_, 1024); - CHECK_ALIGNED(offset_, 4); - encoding = am | offset_ >> 2; - } - encoding |= static_cast<uint32_t>(rn_) << 16; - return encoding; -} - -// Encoding for ARM addressing mode 3. -uint32_t Address::encoding3() const { - const uint32_t offset_mask = (1 << 12) - 1; - uint32_t encoding = encodingArm(); - uint32_t offset = encoding & offset_mask; - CHECK_LT(offset, 256u); - return (encoding & ~offset_mask) | ((offset & 0xf0) << 4) | (offset & 0xf); -} - -// Encoding for vfp load/store addressing. -uint32_t Address::vencoding() const { - CHECK(IsAbsoluteUint<10>(offset_)); // In the range -1020 to +1020. - CHECK_ALIGNED(offset_, 2); // Multiple of 4. - - const uint32_t offset_mask = (1 << 12) - 1; - uint32_t encoding = encodingArm(); - uint32_t offset = encoding & offset_mask; - CHECK((am_ == Offset) || (am_ == NegOffset)); - uint32_t vencoding_value = (encoding & (0xf << kRnShift)) | (offset >> 2); - if (am_ == Offset) { - vencoding_value |= 1 << 23; - } - return vencoding_value; -} - - -bool Address::CanHoldLoadOffsetArm(LoadOperandType type, int offset) { - switch (type) { - case kLoadSignedByte: - case kLoadSignedHalfword: - case kLoadUnsignedHalfword: - case kLoadWordPair: - return IsAbsoluteUint<8>(offset); // Addressing mode 3. - case kLoadUnsignedByte: - case kLoadWord: - return IsAbsoluteUint<12>(offset); // Addressing mode 2. - case kLoadSWord: - case kLoadDWord: - return IsAbsoluteUint<10>(offset); // VFP addressing mode. - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - - -bool Address::CanHoldStoreOffsetArm(StoreOperandType type, int offset) { - switch (type) { - case kStoreHalfword: - case kStoreWordPair: - return IsAbsoluteUint<8>(offset); // Addressing mode 3. - case kStoreByte: - case kStoreWord: - return IsAbsoluteUint<12>(offset); // Addressing mode 2. - case kStoreSWord: - case kStoreDWord: - return IsAbsoluteUint<10>(offset); // VFP addressing mode. - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - -bool Address::CanHoldLoadOffsetThumb(LoadOperandType type, int offset) { - switch (type) { - case kLoadSignedByte: - case kLoadSignedHalfword: - case kLoadUnsignedHalfword: - case kLoadUnsignedByte: - case kLoadWord: - return IsAbsoluteUint<12>(offset); - case kLoadSWord: - case kLoadDWord: - return IsAbsoluteUint<10>(offset) && (offset & 3) == 0; // VFP addressing mode. - case kLoadWordPair: - return IsAbsoluteUint<10>(offset) && (offset & 3) == 0; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - - -bool Address::CanHoldStoreOffsetThumb(StoreOperandType type, int offset) { - switch (type) { - case kStoreHalfword: - case kStoreByte: - case kStoreWord: - return IsAbsoluteUint<12>(offset); - case kStoreSWord: - case kStoreDWord: - return IsAbsoluteUint<10>(offset) && (offset & 3) == 0; // VFP addressing mode. - case kStoreWordPair: - return IsAbsoluteUint<10>(offset) && (offset & 3) == 0; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - -void ArmAssembler::Pad(uint32_t bytes) { - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - for (uint32_t i = 0; i < bytes; ++i) { - buffer_.Emit<uint8_t>(0); - } -} - -static int LeadingZeros(uint32_t val) { - uint32_t alt; - int32_t n; - int32_t count; - - count = 16; - n = 32; - do { - alt = val >> count; - if (alt != 0) { - n = n - count; - val = alt; - } - count >>= 1; - } while (count); - return n - val; -} - - -uint32_t ArmAssembler::ModifiedImmediate(uint32_t value) { - int32_t z_leading; - int32_t z_trailing; - uint32_t b0 = value & 0xff; - - /* Note: case of value==0 must use 0:000:0:0000000 encoding */ - if (value <= 0xFF) - return b0; // 0:000:a:bcdefgh. - if (value == ((b0 << 16) | b0)) - return (0x1 << 12) | b0; /* 0:001:a:bcdefgh */ - if (value == ((b0 << 24) | (b0 << 16) | (b0 << 8) | b0)) - return (0x3 << 12) | b0; /* 0:011:a:bcdefgh */ - b0 = (value >> 8) & 0xff; - if (value == ((b0 << 24) | (b0 << 8))) - return (0x2 << 12) | b0; /* 0:010:a:bcdefgh */ - /* Can we do it with rotation? */ - z_leading = LeadingZeros(value); - z_trailing = 32 - LeadingZeros(~value & (value - 1)); - /* A run of eight or fewer active bits? */ - if ((z_leading + z_trailing) < 24) - return kInvalidModifiedImmediate; /* No - bail */ - /* left-justify the constant, discarding msb (known to be 1) */ - value <<= z_leading + 1; - /* Create bcdefgh */ - value >>= 25; - - /* Put it all together */ - uint32_t v = 8 + z_leading; - - uint32_t i = (v & 16U /* 0b10000 */) >> 4; - uint32_t imm3 = (v >> 1) & 7U /* 0b111 */; - uint32_t a = v & 1; - return value | i << 26 | imm3 << 12 | a << 7; -} - -void ArmAssembler::FinalizeTrackedLabels() { - if (!tracked_labels_.empty()) { - // This array should be sorted, as assembly is generated in linearized order. It isn't - // technically required, but GetAdjustedPosition() used in AdjustLabelPosition() can take - // advantage of it. So ensure that it's actually the case. - DCHECK(std::is_sorted( - tracked_labels_.begin(), - tracked_labels_.end(), - [](const Label* lhs, const Label* rhs) { return lhs->Position() < rhs->Position(); })); - - Label* last_label = nullptr; // Track duplicates, we must not adjust twice. - for (Label* label : tracked_labels_) { - DCHECK_NE(label, last_label); - AdjustLabelPosition(label); - last_label = label; - } - } -} - -} // namespace arm -} // namespace art diff --git a/compiler/utils/arm/assembler_arm.h b/compiler/utils/arm/assembler_arm.h deleted file mode 100644 index 0f24e81be2..0000000000 --- a/compiler/utils/arm/assembler_arm.h +++ /dev/null @@ -1,942 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_UTILS_ARM_ASSEMBLER_ARM_H_ -#define ART_COMPILER_UTILS_ARM_ASSEMBLER_ARM_H_ - -#include <type_traits> -#include <vector> - -#include "base/arena_allocator.h" -#include "base/arena_containers.h" -#include "base/bit_utils.h" -#include "base/enums.h" -#include "base/logging.h" -#include "base/stl_util.h" -#include "base/value_object.h" -#include "constants_arm.h" -#include "utils/arm/assembler_arm_shared.h" -#include "utils/arm/managed_register_arm.h" -#include "utils/assembler.h" -#include "utils/jni_macro_assembler.h" -#include "offsets.h" - -namespace art { -namespace arm { - -class Thumb2Assembler; - -// Assembler literal is a value embedded in code, retrieved using a PC-relative load. -class Literal { - public: - static constexpr size_t kMaxSize = 8; - - Literal(uint32_t size, const uint8_t* data) - : label_(), size_(size) { - DCHECK_LE(size, Literal::kMaxSize); - memcpy(data_, data, size); - } - - template <typename T> - T GetValue() const { - DCHECK_EQ(size_, sizeof(T)); - T value; - memcpy(&value, data_, sizeof(T)); - return value; - } - - uint32_t GetSize() const { - return size_; - } - - const uint8_t* GetData() const { - return data_; - } - - Label* GetLabel() { - return &label_; - } - - const Label* GetLabel() const { - return &label_; - } - - private: - Label label_; - const uint32_t size_; - uint8_t data_[kMaxSize]; - - DISALLOW_COPY_AND_ASSIGN(Literal); -}; - -// Jump table: table of labels emitted after the literals. Similar to literals. -class JumpTable { - public: - explicit JumpTable(std::vector<Label*>&& labels) - : label_(), anchor_label_(), labels_(std::move(labels)) { - } - - uint32_t GetSize() const { - return static_cast<uint32_t>(labels_.size()) * sizeof(uint32_t); - } - - const std::vector<Label*>& GetData() const { - return labels_; - } - - Label* GetLabel() { - return &label_; - } - - const Label* GetLabel() const { - return &label_; - } - - Label* GetAnchorLabel() { - return &anchor_label_; - } - - const Label* GetAnchorLabel() const { - return &anchor_label_; - } - - private: - Label label_; - Label anchor_label_; - std::vector<Label*> labels_; - - DISALLOW_COPY_AND_ASSIGN(JumpTable); -}; - -class ShifterOperand { - public: - ShifterOperand() : type_(kUnknown), rm_(kNoRegister), rs_(kNoRegister), - is_rotate_(false), is_shift_(false), shift_(kNoShift), rotate_(0), immed_(0) { - } - - explicit ShifterOperand(uint32_t immed); - - // Data-processing operands - Register - explicit ShifterOperand(Register rm) : type_(kRegister), rm_(rm), rs_(kNoRegister), - is_rotate_(false), is_shift_(false), shift_(kNoShift), rotate_(0), immed_(0) { - } - - ShifterOperand(uint32_t rotate, uint32_t immed8) : type_(kImmediate), rm_(kNoRegister), - rs_(kNoRegister), - is_rotate_(true), is_shift_(false), shift_(kNoShift), rotate_(rotate), immed_(immed8) { - } - - ShifterOperand(Register rm, Shift shift, uint32_t shift_imm = 0) : type_(kRegister), rm_(rm), - rs_(kNoRegister), - is_rotate_(false), is_shift_(true), shift_(shift), rotate_(0), immed_(shift_imm) { - } - - // Data-processing operands - Logical shift/rotate by register - ShifterOperand(Register rm, Shift shift, Register rs) : type_(kRegister), rm_(rm), - rs_(rs), - is_rotate_(false), is_shift_(true), shift_(shift), rotate_(0), immed_(0) { - } - - bool is_valid() const { return (type_ == kImmediate) || (type_ == kRegister); } - - uint32_t type() const { - CHECK(is_valid()); - return type_; - } - - uint32_t encodingArm() const; - uint32_t encodingThumb() const; - - bool IsEmpty() const { - return type_ == kUnknown; - } - - bool IsImmediate() const { - return type_ == kImmediate; - } - - bool IsRegister() const { - return type_ == kRegister; - } - - bool IsShift() const { - return is_shift_; - } - - uint32_t GetImmediate() const { - return immed_; - } - - Shift GetShift() const { - return shift_; - } - - Register GetRegister() const { - return rm_; - } - - Register GetSecondRegister() const { - return rs_; - } - - enum Type { - kUnknown = -1, - kRegister, - kImmediate - }; - - private: - Type type_; - Register rm_; - Register rs_; - bool is_rotate_; - bool is_shift_; - Shift shift_; - uint32_t rotate_; - uint32_t immed_; - - friend class Thumb2Assembler; - -#ifdef SOURCE_ASSEMBLER_SUPPORT - friend class BinaryAssembler; -#endif -}; - -// Load/store multiple addressing mode. -enum BlockAddressMode { - // bit encoding P U W - DA = (0|0|0) << 21, // decrement after - IA = (0|4|0) << 21, // increment after - DB = (8|0|0) << 21, // decrement before - IB = (8|4|0) << 21, // increment before - DA_W = (0|0|1) << 21, // decrement after with writeback to base - IA_W = (0|4|1) << 21, // increment after with writeback to base - DB_W = (8|0|1) << 21, // decrement before with writeback to base - IB_W = (8|4|1) << 21 // increment before with writeback to base -}; -inline std::ostream& operator<<(std::ostream& os, const BlockAddressMode& rhs) { - os << static_cast<int>(rhs); - return os; -} - -class Address : public ValueObject { - public: - // Memory operand addressing mode (in ARM encoding form. For others we need - // to adjust) - enum Mode { - // bit encoding P U W - Offset = (8|4|0) << 21, // offset (w/o writeback to base) - PreIndex = (8|4|1) << 21, // pre-indexed addressing with writeback - PostIndex = (0|4|0) << 21, // post-indexed addressing with writeback - NegOffset = (8|0|0) << 21, // negative offset (w/o writeback to base) - NegPreIndex = (8|0|1) << 21, // negative pre-indexed with writeback - NegPostIndex = (0|0|0) << 21 // negative post-indexed with writeback - }; - - explicit Address(Register rn, int32_t offset = 0, Mode am = Offset) : rn_(rn), rm_(R0), - offset_(offset), - am_(am), is_immed_offset_(true), shift_(LSL) { - } - - Address(Register rn, Register rm, Mode am = Offset) : rn_(rn), rm_(rm), offset_(0), - am_(am), is_immed_offset_(false), shift_(LSL) { - CHECK_NE(rm, PC); - } - - Address(Register rn, Register rm, Shift shift, uint32_t count, Mode am = Offset) : - rn_(rn), rm_(rm), offset_(count), - am_(am), is_immed_offset_(false), shift_(shift) { - CHECK_NE(rm, PC); - } - - static bool CanHoldLoadOffsetArm(LoadOperandType type, int offset); - static bool CanHoldStoreOffsetArm(StoreOperandType type, int offset); - - static bool CanHoldLoadOffsetThumb(LoadOperandType type, int offset); - static bool CanHoldStoreOffsetThumb(StoreOperandType type, int offset); - - uint32_t encodingArm() const; - uint32_t encodingThumb(bool is_32bit) const; - - uint32_t encoding3() const; - uint32_t vencoding() const; - - uint32_t encodingThumbLdrdStrd() const; - - Register GetRegister() const { - return rn_; - } - - Register GetRegisterOffset() const { - return rm_; - } - - int32_t GetOffset() const { - return offset_; - } - - Mode GetMode() const { - return am_; - } - - bool IsImmediate() const { - return is_immed_offset_; - } - - Shift GetShift() const { - return shift_; - } - - int32_t GetShiftCount() const { - CHECK(!is_immed_offset_); - return offset_; - } - - private: - const Register rn_; - const Register rm_; - const int32_t offset_; // Used as shift amount for register offset. - const Mode am_; - const bool is_immed_offset_; - const Shift shift_; -}; -inline std::ostream& operator<<(std::ostream& os, const Address::Mode& rhs) { - os << static_cast<int>(rhs); - return os; -} - -// Instruction encoding bits. -enum { - H = 1 << 5, // halfword (or byte) - L = 1 << 20, // load (or store) - S = 1 << 20, // set condition code (or leave unchanged) - W = 1 << 21, // writeback base register (or leave unchanged) - A = 1 << 21, // accumulate in multiply instruction (or not) - B = 1 << 22, // unsigned byte (or word) - N = 1 << 22, // long (or short) - U = 1 << 23, // positive (or negative) offset/index - P = 1 << 24, // offset/pre-indexed addressing (or post-indexed addressing) - I = 1 << 25, // immediate shifter operand (or not) - - B0 = 1, - B1 = 1 << 1, - B2 = 1 << 2, - B3 = 1 << 3, - B4 = 1 << 4, - B5 = 1 << 5, - B6 = 1 << 6, - B7 = 1 << 7, - B8 = 1 << 8, - B9 = 1 << 9, - B10 = 1 << 10, - B11 = 1 << 11, - B12 = 1 << 12, - B13 = 1 << 13, - B14 = 1 << 14, - B15 = 1 << 15, - B16 = 1 << 16, - B17 = 1 << 17, - B18 = 1 << 18, - B19 = 1 << 19, - B20 = 1 << 20, - B21 = 1 << 21, - B22 = 1 << 22, - B23 = 1 << 23, - B24 = 1 << 24, - B25 = 1 << 25, - B26 = 1 << 26, - B27 = 1 << 27, - B28 = 1 << 28, - B29 = 1 << 29, - B30 = 1 << 30, - B31 = 1 << 31, - - // Instruction bit masks. - RdMask = 15 << 12, // in str instruction - CondMask = 15 << 28, - CoprocessorMask = 15 << 8, - OpCodeMask = 15 << 21, // in data-processing instructions - Imm24Mask = (1 << 24) - 1, - Off12Mask = (1 << 12) - 1, - - // ldrex/strex register field encodings. - kLdExRnShift = 16, - kLdExRtShift = 12, - kStrExRnShift = 16, - kStrExRdShift = 12, - kStrExRtShift = 0, -}; - -// IfThen state for IT instructions. -enum ItState { - kItOmitted, - kItThen, - kItT = kItThen, - kItElse, - kItE = kItElse -}; - -constexpr uint32_t kNoItCondition = 3; -constexpr uint32_t kInvalidModifiedImmediate = -1; - -extern const char* kRegisterNames[]; -extern const char* kConditionNames[]; - -// This is an abstract ARM assembler. Subclasses provide assemblers for the individual -// instruction sets (ARM32, Thumb2, etc.) -// -class ArmAssembler : public Assembler { - public: - virtual ~ArmAssembler() {} - - // Is this assembler for the thumb instruction set? - virtual bool IsThumb() const = 0; - - // Data-processing instructions. - virtual void and_(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void ands(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - and_(rd, rn, so, cond, kCcSet); - } - - virtual void eor(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void eors(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - eor(rd, rn, so, cond, kCcSet); - } - - virtual void sub(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void subs(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - sub(rd, rn, so, cond, kCcSet); - } - - virtual void rsb(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void rsbs(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - rsb(rd, rn, so, cond, kCcSet); - } - - virtual void add(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void adds(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - add(rd, rn, so, cond, kCcSet); - } - - virtual void adc(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void adcs(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - adc(rd, rn, so, cond, kCcSet); - } - - virtual void sbc(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void sbcs(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - sbc(rd, rn, so, cond, kCcSet); - } - - virtual void rsc(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void rscs(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - rsc(rd, rn, so, cond, kCcSet); - } - - virtual void tst(Register rn, const ShifterOperand& so, Condition cond = AL) = 0; - - virtual void teq(Register rn, const ShifterOperand& so, Condition cond = AL) = 0; - - virtual void cmp(Register rn, const ShifterOperand& so, Condition cond = AL) = 0; - - // Note: CMN updates flags based on addition of its operands. Do not confuse - // the "N" suffix with bitwise inversion performed by MVN. - virtual void cmn(Register rn, const ShifterOperand& so, Condition cond = AL) = 0; - - virtual void orr(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void orrs(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - orr(rd, rn, so, cond, kCcSet); - } - - virtual void orn(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void orns(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - orn(rd, rn, so, cond, kCcSet); - } - - virtual void mov(Register rd, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void movs(Register rd, const ShifterOperand& so, Condition cond = AL) { - mov(rd, so, cond, kCcSet); - } - - virtual void bic(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void bics(Register rd, Register rn, const ShifterOperand& so, Condition cond = AL) { - bic(rd, rn, so, cond, kCcSet); - } - - virtual void mvn(Register rd, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - virtual void mvns(Register rd, const ShifterOperand& so, Condition cond = AL) { - mvn(rd, so, cond, kCcSet); - } - - // Miscellaneous data-processing instructions. - virtual void clz(Register rd, Register rm, Condition cond = AL) = 0; - virtual void movw(Register rd, uint16_t imm16, Condition cond = AL) = 0; - virtual void movt(Register rd, uint16_t imm16, Condition cond = AL) = 0; - virtual void rbit(Register rd, Register rm, Condition cond = AL) = 0; - virtual void rev(Register rd, Register rm, Condition cond = AL) = 0; - virtual void rev16(Register rd, Register rm, Condition cond = AL) = 0; - virtual void revsh(Register rd, Register rm, Condition cond = AL) = 0; - - // Multiply instructions. - virtual void mul(Register rd, Register rn, Register rm, Condition cond = AL) = 0; - virtual void mla(Register rd, Register rn, Register rm, Register ra, - Condition cond = AL) = 0; - virtual void mls(Register rd, Register rn, Register rm, Register ra, - Condition cond = AL) = 0; - virtual void smull(Register rd_lo, Register rd_hi, Register rn, Register rm, - Condition cond = AL) = 0; - virtual void umull(Register rd_lo, Register rd_hi, Register rn, Register rm, - Condition cond = AL) = 0; - - virtual void sdiv(Register rd, Register rn, Register rm, Condition cond = AL) = 0; - virtual void udiv(Register rd, Register rn, Register rm, Condition cond = AL) = 0; - - // Bit field extract instructions. - virtual void sbfx(Register rd, Register rn, uint32_t lsb, uint32_t width, - Condition cond = AL) = 0; - virtual void ubfx(Register rd, Register rn, uint32_t lsb, uint32_t width, - Condition cond = AL) = 0; - - // Load/store instructions. - virtual void ldr(Register rd, const Address& ad, Condition cond = AL) = 0; - virtual void str(Register rd, const Address& ad, Condition cond = AL) = 0; - - virtual void ldrb(Register rd, const Address& ad, Condition cond = AL) = 0; - virtual void strb(Register rd, const Address& ad, Condition cond = AL) = 0; - - virtual void ldrh(Register rd, const Address& ad, Condition cond = AL) = 0; - virtual void strh(Register rd, const Address& ad, Condition cond = AL) = 0; - - virtual void ldrsb(Register rd, const Address& ad, Condition cond = AL) = 0; - virtual void ldrsh(Register rd, const Address& ad, Condition cond = AL) = 0; - - virtual void ldrd(Register rd, const Address& ad, Condition cond = AL) = 0; - virtual void strd(Register rd, const Address& ad, Condition cond = AL) = 0; - - virtual void ldm(BlockAddressMode am, Register base, - RegList regs, Condition cond = AL) = 0; - virtual void stm(BlockAddressMode am, Register base, - RegList regs, Condition cond = AL) = 0; - - virtual void ldrex(Register rd, Register rn, Condition cond = AL) = 0; - virtual void strex(Register rd, Register rt, Register rn, Condition cond = AL) = 0; - virtual void ldrexd(Register rt, Register rt2, Register rn, Condition cond = AL) = 0; - virtual void strexd(Register rd, Register rt, Register rt2, Register rn, Condition cond = AL) = 0; - - // Miscellaneous instructions. - virtual void clrex(Condition cond = AL) = 0; - virtual void nop(Condition cond = AL) = 0; - - // Note that gdb sets breakpoints using the undefined instruction 0xe7f001f0. - virtual void bkpt(uint16_t imm16) = 0; - virtual void svc(uint32_t imm24) = 0; - - virtual void it(Condition firstcond ATTRIBUTE_UNUSED, - ItState i1 ATTRIBUTE_UNUSED = kItOmitted, - ItState i2 ATTRIBUTE_UNUSED = kItOmitted, - ItState i3 ATTRIBUTE_UNUSED = kItOmitted) { - // Ignored if not supported. - } - - virtual void cbz(Register rn, Label* target) = 0; - virtual void cbnz(Register rn, Label* target) = 0; - - // Floating point instructions (VFPv3-D16 and VFPv3-D32 profiles). - virtual void vmovsr(SRegister sn, Register rt, Condition cond = AL) = 0; - virtual void vmovrs(Register rt, SRegister sn, Condition cond = AL) = 0; - virtual void vmovsrr(SRegister sm, Register rt, Register rt2, Condition cond = AL) = 0; - virtual void vmovrrs(Register rt, Register rt2, SRegister sm, Condition cond = AL) = 0; - virtual void vmovdrr(DRegister dm, Register rt, Register rt2, Condition cond = AL) = 0; - virtual void vmovrrd(Register rt, Register rt2, DRegister dm, Condition cond = AL) = 0; - virtual void vmovs(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vmovd(DRegister dd, DRegister dm, Condition cond = AL) = 0; - - // Returns false if the immediate cannot be encoded. - virtual bool vmovs(SRegister sd, float s_imm, Condition cond = AL) = 0; - virtual bool vmovd(DRegister dd, double d_imm, Condition cond = AL) = 0; - - virtual void vldrs(SRegister sd, const Address& ad, Condition cond = AL) = 0; - virtual void vstrs(SRegister sd, const Address& ad, Condition cond = AL) = 0; - virtual void vldrd(DRegister dd, const Address& ad, Condition cond = AL) = 0; - virtual void vstrd(DRegister dd, const Address& ad, Condition cond = AL) = 0; - - virtual void vadds(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) = 0; - virtual void vaddd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) = 0; - virtual void vsubs(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) = 0; - virtual void vsubd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) = 0; - virtual void vmuls(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) = 0; - virtual void vmuld(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) = 0; - virtual void vmlas(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) = 0; - virtual void vmlad(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) = 0; - virtual void vmlss(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) = 0; - virtual void vmlsd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) = 0; - virtual void vdivs(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) = 0; - virtual void vdivd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) = 0; - - virtual void vabss(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vabsd(DRegister dd, DRegister dm, Condition cond = AL) = 0; - virtual void vnegs(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vnegd(DRegister dd, DRegister dm, Condition cond = AL) = 0; - virtual void vsqrts(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vsqrtd(DRegister dd, DRegister dm, Condition cond = AL) = 0; - - virtual void vcvtsd(SRegister sd, DRegister dm, Condition cond = AL) = 0; - virtual void vcvtds(DRegister dd, SRegister sm, Condition cond = AL) = 0; - virtual void vcvtis(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vcvtid(SRegister sd, DRegister dm, Condition cond = AL) = 0; - virtual void vcvtsi(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vcvtdi(DRegister dd, SRegister sm, Condition cond = AL) = 0; - virtual void vcvtus(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vcvtud(SRegister sd, DRegister dm, Condition cond = AL) = 0; - virtual void vcvtsu(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vcvtdu(DRegister dd, SRegister sm, Condition cond = AL) = 0; - - virtual void vcmps(SRegister sd, SRegister sm, Condition cond = AL) = 0; - virtual void vcmpd(DRegister dd, DRegister dm, Condition cond = AL) = 0; - virtual void vcmpsz(SRegister sd, Condition cond = AL) = 0; - virtual void vcmpdz(DRegister dd, Condition cond = AL) = 0; - virtual void vmstat(Condition cond = AL) = 0; // VMRS APSR_nzcv, FPSCR - - virtual void vcntd(DRegister dd, DRegister dm) = 0; - virtual void vpaddld(DRegister dd, DRegister dm, int32_t size, bool is_unsigned) = 0; - - virtual void vpushs(SRegister reg, int nregs, Condition cond = AL) = 0; - virtual void vpushd(DRegister reg, int nregs, Condition cond = AL) = 0; - virtual void vpops(SRegister reg, int nregs, Condition cond = AL) = 0; - virtual void vpopd(DRegister reg, int nregs, Condition cond = AL) = 0; - virtual void vldmiad(Register base_reg, DRegister reg, int nregs, Condition cond = AL) = 0; - virtual void vstmiad(Register base_reg, DRegister reg, int nregs, Condition cond = AL) = 0; - - // Branch instructions. - virtual void b(Label* label, Condition cond = AL) = 0; - virtual void bl(Label* label, Condition cond = AL) = 0; - virtual void blx(Register rm, Condition cond = AL) = 0; - virtual void bx(Register rm, Condition cond = AL) = 0; - - // ADR instruction loading register for branching to the label. - virtual void AdrCode(Register rt, Label* label) = 0; - - // Memory barriers. - virtual void dmb(DmbOptions flavor) = 0; - - void Pad(uint32_t bytes); - - // Adjust label position. - void AdjustLabelPosition(Label* label) { - DCHECK(label->IsBound()); - uint32_t old_position = static_cast<uint32_t>(label->Position()); - uint32_t new_position = GetAdjustedPosition(old_position); - label->Reinitialize(); - DCHECK_GE(static_cast<int>(new_position), 0); - label->BindTo(static_cast<int>(new_position)); - } - - // Get the final position of a label after local fixup based on the old position - // recorded before FinalizeCode(). - virtual uint32_t GetAdjustedPosition(uint32_t old_position) = 0; - - // Macros. - // Most of these are pure virtual as they need to be implemented per instruction set. - - // Create a new literal with a given value. - // NOTE: Force the template parameter to be explicitly specified. - template <typename T> - Literal* NewLiteral(typename Identity<T>::type value) { - static_assert(std::is_integral<T>::value, "T must be an integral type."); - return NewLiteral(sizeof(value), reinterpret_cast<const uint8_t*>(&value)); - } - - // Create a new literal with the given data. - virtual Literal* NewLiteral(size_t size, const uint8_t* data) = 0; - - // Load literal. - virtual void LoadLiteral(Register rt, Literal* literal) = 0; - virtual void LoadLiteral(Register rt, Register rt2, Literal* literal) = 0; - virtual void LoadLiteral(SRegister sd, Literal* literal) = 0; - virtual void LoadLiteral(DRegister dd, Literal* literal) = 0; - - // Add signed constant value to rd. May clobber IP. - virtual void AddConstant(Register rd, Register rn, int32_t value, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - void AddConstantSetFlags(Register rd, Register rn, int32_t value, Condition cond = AL) { - AddConstant(rd, rn, value, cond, kCcSet); - } - void AddConstant(Register rd, int32_t value, Condition cond = AL, SetCc set_cc = kCcDontCare) { - AddConstant(rd, rd, value, cond, set_cc); - } - - virtual void CmpConstant(Register rn, int32_t value, Condition cond = AL) = 0; - - // Load and Store. May clobber IP. - virtual void LoadImmediate(Register rd, int32_t value, Condition cond = AL) = 0; - void LoadSImmediate(SRegister sd, float value, Condition cond = AL) { - if (!vmovs(sd, value, cond)) { - int32_t int_value = bit_cast<int32_t, float>(value); - if (int_value == bit_cast<int32_t, float>(0.0f)) { - // 0.0 is quite common, so we special case it by loading - // 2.0 in `sd` and then substracting it. - bool success = vmovs(sd, 2.0, cond); - CHECK(success); - vsubs(sd, sd, sd, cond); - } else { - LoadImmediate(IP, int_value, cond); - vmovsr(sd, IP, cond); - } - } - } - - virtual void LoadDImmediate(DRegister dd, double value, Condition cond = AL) = 0; - - virtual void MarkExceptionHandler(Label* label) = 0; - virtual void LoadFromOffset(LoadOperandType type, - Register reg, - Register base, - int32_t offset, - Condition cond = AL) = 0; - virtual void StoreToOffset(StoreOperandType type, - Register reg, - Register base, - int32_t offset, - Condition cond = AL) = 0; - virtual void LoadSFromOffset(SRegister reg, - Register base, - int32_t offset, - Condition cond = AL) = 0; - virtual void StoreSToOffset(SRegister reg, - Register base, - int32_t offset, - Condition cond = AL) = 0; - virtual void LoadDFromOffset(DRegister reg, - Register base, - int32_t offset, - Condition cond = AL) = 0; - virtual void StoreDToOffset(DRegister reg, - Register base, - int32_t offset, - Condition cond = AL) = 0; - - virtual void Push(Register rd, Condition cond = AL) = 0; - virtual void Pop(Register rd, Condition cond = AL) = 0; - - virtual void PushList(RegList regs, Condition cond = AL) = 0; - virtual void PopList(RegList regs, Condition cond = AL) = 0; - - virtual void StoreList(RegList regs, size_t stack_offset) = 0; - virtual void LoadList(RegList regs, size_t stack_offset) = 0; - - virtual void Mov(Register rd, Register rm, Condition cond = AL) = 0; - - // Convenience shift instructions. Use mov instruction with shifter operand - // for variants setting the status flags or using a register shift count. - virtual void Lsl(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Lsls(Register rd, Register rm, uint32_t shift_imm, Condition cond = AL) { - Lsl(rd, rm, shift_imm, cond, kCcSet); - } - - virtual void Lsr(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Lsrs(Register rd, Register rm, uint32_t shift_imm, Condition cond = AL) { - Lsr(rd, rm, shift_imm, cond, kCcSet); - } - - virtual void Asr(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Asrs(Register rd, Register rm, uint32_t shift_imm, Condition cond = AL) { - Asr(rd, rm, shift_imm, cond, kCcSet); - } - - virtual void Ror(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Rors(Register rd, Register rm, uint32_t shift_imm, Condition cond = AL) { - Ror(rd, rm, shift_imm, cond, kCcSet); - } - - virtual void Rrx(Register rd, Register rm, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Rrxs(Register rd, Register rm, Condition cond = AL) { - Rrx(rd, rm, cond, kCcSet); - } - - virtual void Lsl(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Lsls(Register rd, Register rm, Register rn, Condition cond = AL) { - Lsl(rd, rm, rn, cond, kCcSet); - } - - virtual void Lsr(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Lsrs(Register rd, Register rm, Register rn, Condition cond = AL) { - Lsr(rd, rm, rn, cond, kCcSet); - } - - virtual void Asr(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Asrs(Register rd, Register rm, Register rn, Condition cond = AL) { - Asr(rd, rm, rn, cond, kCcSet); - } - - virtual void Ror(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) = 0; - - void Rors(Register rd, Register rm, Register rn, Condition cond = AL) { - Ror(rd, rm, rn, cond, kCcSet); - } - - // Returns whether the `immediate` can fit in a `ShifterOperand`. If yes, - // `shifter_op` contains the operand. - virtual bool ShifterOperandCanHold(Register rd, - Register rn, - Opcode opcode, - uint32_t immediate, - SetCc set_cc, - ShifterOperand* shifter_op) = 0; - bool ShifterOperandCanHold(Register rd, - Register rn, - Opcode opcode, - uint32_t immediate, - ShifterOperand* shifter_op) { - return ShifterOperandCanHold(rd, rn, opcode, immediate, kCcDontCare, shifter_op); - } - - virtual bool ShifterOperandCanAlwaysHold(uint32_t immediate) = 0; - - static bool IsInstructionForExceptionHandling(uintptr_t pc); - - virtual void CompareAndBranchIfZero(Register r, Label* label) = 0; - virtual void CompareAndBranchIfNonZero(Register r, Label* label) = 0; - - static uint32_t ModifiedImmediate(uint32_t value); - - static bool IsLowRegister(Register r) { - return r < R8; - } - - static bool IsHighRegister(Register r) { - return r >= R8; - } - - // - // Heap poisoning. - // - - // Poison a heap reference contained in `reg`. - void PoisonHeapReference(Register reg) { - // reg = -reg. - rsb(reg, reg, ShifterOperand(0)); - } - // Unpoison a heap reference contained in `reg`. - void UnpoisonHeapReference(Register reg) { - // reg = -reg. - rsb(reg, reg, ShifterOperand(0)); - } - // Poison a heap reference contained in `reg` if heap poisoning is enabled. - void MaybePoisonHeapReference(Register reg) { - if (kPoisonHeapReferences) { - PoisonHeapReference(reg); - } - } - // Unpoison a heap reference contained in `reg` if heap poisoning is enabled. - void MaybeUnpoisonHeapReference(Register reg) { - if (kPoisonHeapReferences) { - UnpoisonHeapReference(reg); - } - } - - void Jump(Label* label) OVERRIDE { - b(label); - } - - // Jump table support. This is split into three functions: - // - // * CreateJumpTable creates the internal metadata to track the jump targets, and emits code to - // load the base address of the jump table. - // - // * EmitJumpTableDispatch emits the code to actually jump, assuming that the right table value - // has been loaded into a register already. - // - // * FinalizeTables emits the jump table into the literal pool. This can only be called after the - // labels for the jump targets have been finalized. - - // Create a jump table for the given labels that will be emitted when finalizing. Create a load - // sequence (or placeholder) that stores the base address into the given register. When the table - // is emitted, offsets will be relative to the location EmitJumpTableDispatch was called on (the - // anchor). - virtual JumpTable* CreateJumpTable(std::vector<Label*>&& labels, Register base_reg) = 0; - - // Emit the jump-table jump, assuming that the right value was loaded into displacement_reg. - virtual void EmitJumpTableDispatch(JumpTable* jump_table, Register displacement_reg) = 0; - - // Bind a Label that needs to be updated by the assembler in FinalizeCode() if its position - // changes due to branch/literal fixup. - void BindTrackedLabel(Label* label) { - Bind(label); - tracked_labels_.push_back(label); - } - - protected: - explicit ArmAssembler(ArenaAllocator* arena) - : Assembler(arena), tracked_labels_(arena->Adapter(kArenaAllocAssembler)) {} - - // Returns whether or not the given register is used for passing parameters. - static int RegisterCompare(const Register* reg1, const Register* reg2) { - return *reg1 - *reg2; - } - - void FinalizeTrackedLabels(); - - // Tracked labels. Use a vector, as we need to sort before adjusting. - ArenaVector<Label*> tracked_labels_; -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_UTILS_ARM_ASSEMBLER_ARM_H_ diff --git a/compiler/utils/arm/assembler_arm_vixl.cc b/compiler/utils/arm/assembler_arm_vixl.cc index 6afc3ddecb..af3b4474e3 100644 --- a/compiler/utils/arm/assembler_arm_vixl.cc +++ b/compiler/utils/arm/assembler_arm_vixl.cc @@ -18,6 +18,8 @@ #include <type_traits> #include "assembler_arm_vixl.h" +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "entrypoints/quick/quick_entrypoints.h" #include "thread.h" @@ -35,7 +37,10 @@ namespace arm { #define ___ vixl_masm_. #endif +// Thread register definition. extern const vixl32::Register tr(TR); +// Marking register definition. +extern const vixl32::Register mr(MR); void ArmVIXLAssembler::FinalizeCode() { vixl_masm_.FinalizeCode(); diff --git a/compiler/utils/arm/assembler_arm_vixl.h b/compiler/utils/arm/assembler_arm_vixl.h index e81e767575..66b22ea87c 100644 --- a/compiler/utils/arm/assembler_arm_vixl.h +++ b/compiler/utils/arm/assembler_arm_vixl.h @@ -241,6 +241,8 @@ class ArmVIXLAssembler FINAL : public Assembler { // Thread register declaration. extern const vixl32::Register tr; +// Marking register declaration. +extern const vixl32::Register mr; } // namespace arm } // namespace art diff --git a/compiler/utils/arm/assembler_thumb2.cc b/compiler/utils/arm/assembler_thumb2.cc deleted file mode 100644 index d7096b3c87..0000000000 --- a/compiler/utils/arm/assembler_thumb2.cc +++ /dev/null @@ -1,4076 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <type_traits> - -#include "assembler_thumb2.h" - -#include "base/bit_utils.h" -#include "base/logging.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "offsets.h" -#include "thread.h" - -namespace art { -namespace arm { - -template <typename Function> -void Thumb2Assembler::Fixup::ForExpandableDependencies(Thumb2Assembler* assembler, Function fn) { - static_assert( - std::is_same<typename std::result_of<Function(FixupId, FixupId)>::type, void>::value, - "Incorrect signature for argument `fn`: expected (FixupId, FixupId) -> void"); - Fixup* fixups = assembler->fixups_.data(); - for (FixupId fixup_id = 0u, end_id = assembler->fixups_.size(); fixup_id != end_id; ++fixup_id) { - uint32_t target = fixups[fixup_id].target_; - if (target > fixups[fixup_id].location_) { - for (FixupId id = fixup_id + 1u; id != end_id && fixups[id].location_ < target; ++id) { - if (fixups[id].CanExpand()) { - fn(id, fixup_id); - } - } - } else { - for (FixupId id = fixup_id; id != 0u && fixups[id - 1u].location_ >= target; --id) { - if (fixups[id - 1u].CanExpand()) { - fn(id - 1u, fixup_id); - } - } - } - } -} - -void Thumb2Assembler::Fixup::PrepareDependents(Thumb2Assembler* assembler) { - // For each Fixup, it's easy to find the Fixups that it depends on as they are either - // the following or the preceding Fixups until we find the target. However, for fixup - // adjustment we need the reverse lookup, i.e. what Fixups depend on a given Fixup. - // This function creates a compact representation of this relationship, where we have - // all the dependents in a single array and Fixups reference their ranges by start - // index and count. (Instead of having a per-fixup vector.) - - // Count the number of dependents of each Fixup. - Fixup* fixups = assembler->fixups_.data(); - ForExpandableDependencies( - assembler, - [fixups](FixupId dependency, FixupId dependent ATTRIBUTE_UNUSED) { - fixups[dependency].dependents_count_ += 1u; - }); - // Assign index ranges in fixup_dependents_ to individual fixups. Record the end of the - // range in dependents_start_, we shall later decrement it as we fill in fixup_dependents_. - uint32_t number_of_dependents = 0u; - for (FixupId fixup_id = 0u, end_id = assembler->fixups_.size(); fixup_id != end_id; ++fixup_id) { - number_of_dependents += fixups[fixup_id].dependents_count_; - fixups[fixup_id].dependents_start_ = number_of_dependents; - } - if (number_of_dependents == 0u) { - return; - } - // Create and fill in the fixup_dependents_. - assembler->fixup_dependents_.resize(number_of_dependents); - FixupId* dependents = assembler->fixup_dependents_.data(); - ForExpandableDependencies( - assembler, - [fixups, dependents](FixupId dependency, FixupId dependent) { - fixups[dependency].dependents_start_ -= 1u; - dependents[fixups[dependency].dependents_start_] = dependent; - }); -} - -void Thumb2Assembler::BindLabel(Label* label, uint32_t bound_pc) { - CHECK(!label->IsBound()); - - while (label->IsLinked()) { - FixupId fixup_id = label->Position(); // The id for linked Fixup. - Fixup* fixup = GetFixup(fixup_id); // Get the Fixup at this id. - fixup->Resolve(bound_pc); // Fixup can be resolved now. - uint32_t fixup_location = fixup->GetLocation(); - uint16_t next = buffer_.Load<uint16_t>(fixup_location); // Get next in chain. - buffer_.Store<int16_t>(fixup_location, 0); - label->position_ = next; // Move to next. - } - label->BindTo(bound_pc); -} - -uint32_t Thumb2Assembler::BindLiterals() { - // We don't add the padding here, that's done only after adjusting the Fixup sizes. - uint32_t code_size = buffer_.Size(); - for (Literal& lit : literals_) { - Label* label = lit.GetLabel(); - BindLabel(label, code_size); - code_size += lit.GetSize(); - } - return code_size; -} - -void Thumb2Assembler::BindJumpTables(uint32_t code_size) { - for (JumpTable& table : jump_tables_) { - Label* label = table.GetLabel(); - BindLabel(label, code_size); - code_size += table.GetSize(); - } -} - -void Thumb2Assembler::AdjustFixupIfNeeded(Fixup* fixup, uint32_t* current_code_size, - std::deque<FixupId>* fixups_to_recalculate) { - uint32_t adjustment = fixup->AdjustSizeIfNeeded(*current_code_size); - if (adjustment != 0u) { - DCHECK(fixup->CanExpand()); - *current_code_size += adjustment; - for (FixupId dependent_id : fixup->Dependents(*this)) { - Fixup* dependent = GetFixup(dependent_id); - dependent->IncreaseAdjustment(adjustment); - if (buffer_.Load<int16_t>(dependent->GetLocation()) == 0) { - buffer_.Store<int16_t>(dependent->GetLocation(), 1); - fixups_to_recalculate->push_back(dependent_id); - } - } - } -} - -uint32_t Thumb2Assembler::AdjustFixups() { - Fixup::PrepareDependents(this); - uint32_t current_code_size = buffer_.Size(); - std::deque<FixupId> fixups_to_recalculate; - if (kIsDebugBuild) { - // We will use the placeholders in the buffer_ to mark whether the fixup has - // been added to the fixups_to_recalculate. Make sure we start with zeros. - for (Fixup& fixup : fixups_) { - CHECK_EQ(buffer_.Load<int16_t>(fixup.GetLocation()), 0); - } - } - for (Fixup& fixup : fixups_) { - AdjustFixupIfNeeded(&fixup, ¤t_code_size, &fixups_to_recalculate); - } - while (!fixups_to_recalculate.empty()) { - do { - // Pop the fixup. - FixupId fixup_id = fixups_to_recalculate.front(); - fixups_to_recalculate.pop_front(); - Fixup* fixup = GetFixup(fixup_id); - DCHECK_NE(buffer_.Load<int16_t>(fixup->GetLocation()), 0); - buffer_.Store<int16_t>(fixup->GetLocation(), 0); - // See if it needs adjustment. - AdjustFixupIfNeeded(fixup, ¤t_code_size, &fixups_to_recalculate); - } while (!fixups_to_recalculate.empty()); - - if ((current_code_size & 2) != 0 && (!literals_.empty() || !jump_tables_.empty())) { - // If we need to add padding before literals, this may just push some out of range, - // so recalculate all load literals. This makes up for the fact that we don't mark - // load literal as a dependency of all previous Fixups even though it actually is. - for (Fixup& fixup : fixups_) { - if (fixup.IsLoadLiteral()) { - AdjustFixupIfNeeded(&fixup, ¤t_code_size, &fixups_to_recalculate); - } - } - } - } - if (kIsDebugBuild) { - // Check that no fixup is marked as being in fixups_to_recalculate anymore. - for (Fixup& fixup : fixups_) { - CHECK_EQ(buffer_.Load<int16_t>(fixup.GetLocation()), 0); - } - } - - // Adjust literal pool labels for padding. - DCHECK_ALIGNED(current_code_size, 2); - uint32_t literals_adjustment = current_code_size + (current_code_size & 2) - buffer_.Size(); - if (literals_adjustment != 0u) { - for (Literal& literal : literals_) { - Label* label = literal.GetLabel(); - DCHECK(label->IsBound()); - int old_position = label->Position(); - label->Reinitialize(); - label->BindTo(old_position + literals_adjustment); - } - for (JumpTable& table : jump_tables_) { - Label* label = table.GetLabel(); - DCHECK(label->IsBound()); - int old_position = label->Position(); - label->Reinitialize(); - label->BindTo(old_position + literals_adjustment); - } - } - - return current_code_size; -} - -void Thumb2Assembler::EmitFixups(uint32_t adjusted_code_size) { - // Move non-fixup code to its final place and emit fixups. - // Process fixups in reverse order so that we don't repeatedly move the same data. - size_t src_end = buffer_.Size(); - size_t dest_end = adjusted_code_size; - buffer_.Resize(dest_end); - DCHECK_GE(dest_end, src_end); - for (auto i = fixups_.rbegin(), end = fixups_.rend(); i != end; ++i) { - Fixup* fixup = &*i; - size_t old_fixup_location = fixup->GetLocation(); - if (fixup->GetOriginalSize() == fixup->GetSize()) { - // The size of this Fixup didn't change. To avoid moving the data - // in small chunks, emit the code to its original position. - fixup->Finalize(dest_end - src_end); - fixup->Emit(old_fixup_location, &buffer_, adjusted_code_size); - } else { - // Move the data between the end of the fixup and src_end to its final location. - size_t src_begin = old_fixup_location + fixup->GetOriginalSizeInBytes(); - size_t data_size = src_end - src_begin; - size_t dest_begin = dest_end - data_size; - buffer_.Move(dest_begin, src_begin, data_size); - src_end = old_fixup_location; - dest_end = dest_begin - fixup->GetSizeInBytes(); - // Finalize the Fixup and emit the data to the new location. - fixup->Finalize(dest_end - src_end); - fixup->Emit(fixup->GetLocation(), &buffer_, adjusted_code_size); - } - } - CHECK_EQ(src_end, dest_end); -} - -void Thumb2Assembler::EmitLiterals() { - if (!literals_.empty()) { - // Load literal instructions (LDR, LDRD, VLDR) require 4-byte alignment. - // We don't support byte and half-word literals. - uint32_t code_size = buffer_.Size(); - DCHECK_ALIGNED(code_size, 2); - if ((code_size & 2u) != 0u) { - Emit16(0); - } - for (Literal& literal : literals_) { - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - DCHECK_EQ(static_cast<size_t>(literal.GetLabel()->Position()), buffer_.Size()); - DCHECK(literal.GetSize() == 4u || literal.GetSize() == 8u); - for (size_t i = 0, size = literal.GetSize(); i != size; ++i) { - buffer_.Emit<uint8_t>(literal.GetData()[i]); - } - } - } -} - -void Thumb2Assembler::EmitJumpTables() { - if (!jump_tables_.empty()) { - // Jump tables require 4 byte alignment. (We don't support byte and half-word jump tables.) - uint32_t code_size = buffer_.Size(); - DCHECK_ALIGNED(code_size, 2); - if ((code_size & 2u) != 0u) { - Emit16(0); - } - for (JumpTable& table : jump_tables_) { - // Bulk ensure capacity, as this may be large. - size_t orig_size = buffer_.Size(); - size_t required_capacity = orig_size + table.GetSize(); - if (required_capacity > buffer_.Capacity()) { - buffer_.ExtendCapacity(required_capacity); - } -#ifndef NDEBUG - buffer_.has_ensured_capacity_ = true; -#endif - - DCHECK_EQ(static_cast<size_t>(table.GetLabel()->Position()), buffer_.Size()); - int32_t anchor_position = table.GetAnchorLabel()->Position() + 4; - - for (Label* target : table.GetData()) { - // Ensure that the label was tracked, so that it will have the right position. - DCHECK(std::find(tracked_labels_.begin(), tracked_labels_.end(), target) != - tracked_labels_.end()); - - int32_t offset = target->Position() - anchor_position; - buffer_.Emit<int32_t>(offset); - } - -#ifndef NDEBUG - buffer_.has_ensured_capacity_ = false; -#endif - size_t new_size = buffer_.Size(); - DCHECK_LE(new_size - orig_size, table.GetSize()); - } - } -} - -void Thumb2Assembler::PatchCFI() { - if (cfi().NumberOfDelayedAdvancePCs() == 0u) { - return; - } - - typedef DebugFrameOpCodeWriterForAssembler::DelayedAdvancePC DelayedAdvancePC; - const auto data = cfi().ReleaseStreamAndPrepareForDelayedAdvancePC(); - const std::vector<uint8_t>& old_stream = data.first; - const std::vector<DelayedAdvancePC>& advances = data.second; - - // Refill our data buffer with patched opcodes. - cfi().ReserveCFIStream(old_stream.size() + advances.size() + 16); - size_t stream_pos = 0; - for (const DelayedAdvancePC& advance : advances) { - DCHECK_GE(advance.stream_pos, stream_pos); - // Copy old data up to the point where advance was issued. - cfi().AppendRawData(old_stream, stream_pos, advance.stream_pos); - stream_pos = advance.stream_pos; - // Insert the advance command with its final offset. - size_t final_pc = GetAdjustedPosition(advance.pc); - cfi().AdvancePC(final_pc); - } - // Copy the final segment if any. - cfi().AppendRawData(old_stream, stream_pos, old_stream.size()); -} - -inline int16_t Thumb2Assembler::BEncoding16(int32_t offset, Condition cond) { - DCHECK_ALIGNED(offset, 2); - int16_t encoding = B15 | B14; - if (cond != AL) { - DCHECK(IsInt<9>(offset)); - encoding |= B12 | (static_cast<int32_t>(cond) << 8) | ((offset >> 1) & 0xff); - } else { - DCHECK(IsInt<12>(offset)); - encoding |= B13 | ((offset >> 1) & 0x7ff); - } - return encoding; -} - -inline int32_t Thumb2Assembler::BEncoding32(int32_t offset, Condition cond) { - DCHECK_ALIGNED(offset, 2); - int32_t s = (offset >> 31) & 1; // Sign bit. - int32_t encoding = B31 | B30 | B29 | B28 | B15 | - (s << 26) | // Sign bit goes to bit 26. - ((offset >> 1) & 0x7ff); // imm11 goes to bits 0-10. - if (cond != AL) { - DCHECK(IsInt<21>(offset)); - // Encode cond, move imm6 from bits 12-17 to bits 16-21 and move J1 and J2. - encoding |= (static_cast<int32_t>(cond) << 22) | ((offset & 0x3f000) << (16 - 12)) | - ((offset & (1 << 19)) >> (19 - 13)) | // Extract J1 from bit 19 to bit 13. - ((offset & (1 << 18)) >> (18 - 11)); // Extract J2 from bit 18 to bit 11. - } else { - DCHECK(IsInt<25>(offset)); - int32_t j1 = ((offset >> 23) ^ s ^ 1) & 1; // Calculate J1 from I1 extracted from bit 23. - int32_t j2 = ((offset >> 22)^ s ^ 1) & 1; // Calculate J2 from I2 extracted from bit 22. - // Move imm10 from bits 12-21 to bits 16-25 and add J1 and J2. - encoding |= B12 | ((offset & 0x3ff000) << (16 - 12)) | - (j1 << 13) | (j2 << 11); - } - return encoding; -} - -inline int16_t Thumb2Assembler::CbxzEncoding16(Register rn, int32_t offset, Condition cond) { - DCHECK(!IsHighRegister(rn)); - DCHECK_ALIGNED(offset, 2); - DCHECK(IsUint<7>(offset)); - DCHECK(cond == EQ || cond == NE); - return B15 | B13 | B12 | B8 | (cond == NE ? B11 : 0) | static_cast<int32_t>(rn) | - ((offset & 0x3e) << (3 - 1)) | // Move imm5 from bits 1-5 to bits 3-7. - ((offset & 0x40) << (9 - 6)); // Move i from bit 6 to bit 11 -} - -inline int16_t Thumb2Assembler::CmpRnImm8Encoding16(Register rn, int32_t value) { - DCHECK(!IsHighRegister(rn)); - DCHECK(IsUint<8>(value)); - return B13 | B11 | (rn << 8) | value; -} - -inline int16_t Thumb2Assembler::AddRdnRmEncoding16(Register rdn, Register rm) { - // The high bit of rn is moved across 4-bit rm. - return B14 | B10 | (static_cast<int32_t>(rm) << 3) | - (static_cast<int32_t>(rdn) & 7) | ((static_cast<int32_t>(rdn) & 8) << 4); -} - -inline int32_t Thumb2Assembler::MovwEncoding32(Register rd, int32_t value) { - DCHECK(IsUint<16>(value)); - return B31 | B30 | B29 | B28 | B25 | B22 | - (static_cast<int32_t>(rd) << 8) | - ((value & 0xf000) << (16 - 12)) | // Move imm4 from bits 12-15 to bits 16-19. - ((value & 0x0800) << (26 - 11)) | // Move i from bit 11 to bit 26. - ((value & 0x0700) << (12 - 8)) | // Move imm3 from bits 8-10 to bits 12-14. - (value & 0xff); // Keep imm8 in bits 0-7. -} - -inline int32_t Thumb2Assembler::MovtEncoding32(Register rd, int32_t value) { - DCHECK_EQ(value & 0xffff, 0); - int32_t movw_encoding = MovwEncoding32(rd, (value >> 16) & 0xffff); - return movw_encoding | B25 | B23; -} - -inline int32_t Thumb2Assembler::MovModImmEncoding32(Register rd, int32_t value) { - uint32_t mod_imm = ModifiedImmediate(value); - DCHECK_NE(mod_imm, kInvalidModifiedImmediate); - return B31 | B30 | B29 | B28 | B22 | B19 | B18 | B17 | B16 | - (static_cast<int32_t>(rd) << 8) | static_cast<int32_t>(mod_imm); -} - -inline int16_t Thumb2Assembler::LdrLitEncoding16(Register rt, int32_t offset) { - DCHECK(!IsHighRegister(rt)); - DCHECK_ALIGNED(offset, 4); - DCHECK(IsUint<10>(offset)); - return B14 | B11 | (static_cast<int32_t>(rt) << 8) | (offset >> 2); -} - -inline int32_t Thumb2Assembler::LdrLitEncoding32(Register rt, int32_t offset) { - // NOTE: We don't support negative offset, i.e. U=0 (B23). - return LdrRtRnImm12Encoding(rt, PC, offset); -} - -inline int32_t Thumb2Assembler::LdrdEncoding32(Register rt, Register rt2, Register rn, int32_t offset) { - DCHECK_ALIGNED(offset, 4); - CHECK(IsUint<10>(offset)); - return B31 | B30 | B29 | B27 | - B24 /* P = 1 */ | B23 /* U = 1 */ | B22 | 0 /* W = 0 */ | B20 | - (static_cast<int32_t>(rn) << 16) | (static_cast<int32_t>(rt) << 12) | - (static_cast<int32_t>(rt2) << 8) | (offset >> 2); -} - -inline int32_t Thumb2Assembler::VldrsEncoding32(SRegister sd, Register rn, int32_t offset) { - DCHECK_ALIGNED(offset, 4); - CHECK(IsUint<10>(offset)); - return B31 | B30 | B29 | B27 | B26 | B24 | - B23 /* U = 1 */ | B20 | B11 | B9 | - (static_cast<int32_t>(rn) << 16) | - ((static_cast<int32_t>(sd) & 0x01) << (22 - 0)) | // Move D from bit 0 to bit 22. - ((static_cast<int32_t>(sd) & 0x1e) << (12 - 1)) | // Move Vd from bits 1-4 to bits 12-15. - (offset >> 2); -} - -inline int32_t Thumb2Assembler::VldrdEncoding32(DRegister dd, Register rn, int32_t offset) { - DCHECK_ALIGNED(offset, 4); - CHECK(IsUint<10>(offset)); - return B31 | B30 | B29 | B27 | B26 | B24 | - B23 /* U = 1 */ | B20 | B11 | B9 | B8 | - (rn << 16) | - ((static_cast<int32_t>(dd) & 0x10) << (22 - 4)) | // Move D from bit 4 to bit 22. - ((static_cast<int32_t>(dd) & 0x0f) << (12 - 0)) | // Move Vd from bits 0-3 to bits 12-15. - (offset >> 2); -} - -inline int16_t Thumb2Assembler::LdrRtRnImm5Encoding16(Register rt, Register rn, int32_t offset) { - DCHECK(!IsHighRegister(rt)); - DCHECK(!IsHighRegister(rn)); - DCHECK_ALIGNED(offset, 4); - DCHECK(IsUint<7>(offset)); - return B14 | B13 | B11 | - (static_cast<int32_t>(rn) << 3) | static_cast<int32_t>(rt) | - (offset << (6 - 2)); // Move imm5 from bits 2-6 to bits 6-10. -} - -int32_t Thumb2Assembler::Fixup::LoadWideOrFpEncoding(Register rbase, int32_t offset) const { - switch (type_) { - case kLoadLiteralWide: - return LdrdEncoding32(rn_, rt2_, rbase, offset); - case kLoadFPLiteralSingle: - return VldrsEncoding32(sd_, rbase, offset); - case kLoadFPLiteralDouble: - return VldrdEncoding32(dd_, rbase, offset); - default: - LOG(FATAL) << "Unexpected type: " << static_cast<int>(type_); - UNREACHABLE(); - } -} - -inline int32_t Thumb2Assembler::LdrRtRnImm12Encoding(Register rt, Register rn, int32_t offset) { - DCHECK(IsUint<12>(offset)); - return B31 | B30 | B29 | B28 | B27 | B23 | B22 | B20 | (rn << 16) | (rt << 12) | offset; -} - -inline int16_t Thumb2Assembler::AdrEncoding16(Register rd, int32_t offset) { - DCHECK(IsUint<10>(offset)); - DCHECK(IsAligned<4>(offset)); - DCHECK(!IsHighRegister(rd)); - return B15 | B13 | (rd << 8) | (offset >> 2); -} - -inline int32_t Thumb2Assembler::AdrEncoding32(Register rd, int32_t offset) { - DCHECK(IsUint<12>(offset)); - // Bit 26: offset[11] - // Bits 14-12: offset[10-8] - // Bits 7-0: offset[7-0] - int32_t immediate_mask = - ((offset & (1 << 11)) << (26 - 11)) | - ((offset & (7 << 8)) << (12 - 8)) | - (offset & 0xFF); - return B31 | B30 | B29 | B28 | B25 | B19 | B18 | B17 | B16 | (rd << 8) | immediate_mask; -} - -void Thumb2Assembler::FinalizeCode() { - ArmAssembler::FinalizeCode(); - uint32_t size_after_literals = BindLiterals(); - BindJumpTables(size_after_literals); - uint32_t adjusted_code_size = AdjustFixups(); - EmitFixups(adjusted_code_size); - EmitLiterals(); - FinalizeTrackedLabels(); - EmitJumpTables(); - PatchCFI(); -} - -bool Thumb2Assembler::ShifterOperandCanAlwaysHold(uint32_t immediate) { - return ArmAssembler::ModifiedImmediate(immediate) != kInvalidModifiedImmediate; -} - -bool Thumb2Assembler::ShifterOperandCanHold(Register rd ATTRIBUTE_UNUSED, - Register rn ATTRIBUTE_UNUSED, - Opcode opcode, - uint32_t immediate, - SetCc set_cc, - ShifterOperand* shifter_op) { - shifter_op->type_ = ShifterOperand::kImmediate; - shifter_op->immed_ = immediate; - shifter_op->is_shift_ = false; - shifter_op->is_rotate_ = false; - switch (opcode) { - case ADD: - case SUB: - // Less than (or equal to) 12 bits can be done if we don't need to set condition codes. - if (immediate < (1 << 12) && set_cc != kCcSet) { - return true; - } - return ArmAssembler::ModifiedImmediate(immediate) != kInvalidModifiedImmediate; - - case MOV: - // TODO: Support less than or equal to 12bits. - return ArmAssembler::ModifiedImmediate(immediate) != kInvalidModifiedImmediate; - - case MVN: - default: - return ArmAssembler::ModifiedImmediate(immediate) != kInvalidModifiedImmediate; - } -} - -void Thumb2Assembler::and_(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, AND, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::eor(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, EOR, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::sub(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, SUB, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::rsb(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, RSB, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::add(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, ADD, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::adc(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, ADC, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::sbc(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, SBC, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::rsc(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, RSC, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::tst(Register rn, const ShifterOperand& so, Condition cond) { - CHECK_NE(rn, PC); // Reserve tst pc instruction for exception handler marker. - EmitDataProcessing(cond, TST, kCcSet, rn, R0, so); -} - - -void Thumb2Assembler::teq(Register rn, const ShifterOperand& so, Condition cond) { - CHECK_NE(rn, PC); // Reserve teq pc instruction for exception handler marker. - EmitDataProcessing(cond, TEQ, kCcSet, rn, R0, so); -} - - -void Thumb2Assembler::cmp(Register rn, const ShifterOperand& so, Condition cond) { - EmitDataProcessing(cond, CMP, kCcSet, rn, R0, so); -} - - -void Thumb2Assembler::cmn(Register rn, const ShifterOperand& so, Condition cond) { - EmitDataProcessing(cond, CMN, kCcSet, rn, R0, so); -} - - -void Thumb2Assembler::orr(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, ORR, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::orn(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, ORN, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::mov(Register rd, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, MOV, set_cc, R0, rd, so); -} - - -void Thumb2Assembler::bic(Register rd, Register rn, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, BIC, set_cc, rn, rd, so); -} - - -void Thumb2Assembler::mvn(Register rd, const ShifterOperand& so, - Condition cond, SetCc set_cc) { - EmitDataProcessing(cond, MVN, set_cc, R0, rd, so); -} - - -void Thumb2Assembler::mul(Register rd, Register rn, Register rm, Condition cond) { - CheckCondition(cond); - - if (rd == rm && !IsHighRegister(rd) && !IsHighRegister(rn) && !force_32bit_) { - // 16 bit. - int16_t encoding = B14 | B9 | B8 | B6 | - rn << 3 | rd; - Emit16(encoding); - } else { - // 32 bit. - uint32_t op1 = 0U /* 0b000 */; - uint32_t op2 = 0U /* 0b00 */; - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B24 | - op1 << 20 | - B15 | B14 | B13 | B12 | - op2 << 4 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rm); - - Emit32(encoding); - } -} - - -void Thumb2Assembler::mla(Register rd, Register rn, Register rm, Register ra, - Condition cond) { - CheckCondition(cond); - - uint32_t op1 = 0U /* 0b000 */; - uint32_t op2 = 0U /* 0b00 */; - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B24 | - op1 << 20 | - op2 << 4 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(ra) << 12 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rm); - - Emit32(encoding); -} - - -void Thumb2Assembler::mls(Register rd, Register rn, Register rm, Register ra, - Condition cond) { - CheckCondition(cond); - - uint32_t op1 = 0U /* 0b000 */; - uint32_t op2 = 01 /* 0b01 */; - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B24 | - op1 << 20 | - op2 << 4 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(ra) << 12 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rm); - - Emit32(encoding); -} - - -void Thumb2Assembler::smull(Register rd_lo, Register rd_hi, Register rn, - Register rm, Condition cond) { - CheckCondition(cond); - - uint32_t op1 = 0U /* 0b000; */; - uint32_t op2 = 0U /* 0b0000 */; - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B24 | B23 | - op1 << 20 | - op2 << 4 | - static_cast<uint32_t>(rd_lo) << 12 | - static_cast<uint32_t>(rd_hi) << 8 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rm); - - Emit32(encoding); -} - - -void Thumb2Assembler::umull(Register rd_lo, Register rd_hi, Register rn, - Register rm, Condition cond) { - CheckCondition(cond); - - uint32_t op1 = 2U /* 0b010; */; - uint32_t op2 = 0U /* 0b0000 */; - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B24 | B23 | - op1 << 20 | - op2 << 4 | - static_cast<uint32_t>(rd_lo) << 12 | - static_cast<uint32_t>(rd_hi) << 8 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rm); - - Emit32(encoding); -} - - -void Thumb2Assembler::sdiv(Register rd, Register rn, Register rm, Condition cond) { - CheckCondition(cond); - - uint32_t op1 = 1U /* 0b001 */; - uint32_t op2 = 15U /* 0b1111 */; - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B24 | B23 | B20 | - op1 << 20 | - op2 << 4 | - 0xf << 12 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rm); - - Emit32(encoding); -} - - -void Thumb2Assembler::udiv(Register rd, Register rn, Register rm, Condition cond) { - CheckCondition(cond); - - uint32_t op1 = 1U /* 0b001 */; - uint32_t op2 = 15U /* 0b1111 */; - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B24 | B23 | B21 | B20 | - op1 << 20 | - op2 << 4 | - 0xf << 12 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rm); - - Emit32(encoding); -} - - -void Thumb2Assembler::sbfx(Register rd, Register rn, uint32_t lsb, uint32_t width, Condition cond) { - CheckCondition(cond); - CHECK_LE(lsb, 31U); - CHECK(1U <= width && width <= 32U) << width; - uint32_t widthminus1 = width - 1; - uint32_t imm2 = lsb & (B1 | B0); // Bits 0-1 of `lsb`. - uint32_t imm3 = (lsb & (B4 | B3 | B2)) >> 2; // Bits 2-4 of `lsb`. - - uint32_t op = 20U /* 0b10100 */; - int32_t encoding = B31 | B30 | B29 | B28 | B25 | - op << 20 | - static_cast<uint32_t>(rn) << 16 | - imm3 << 12 | - static_cast<uint32_t>(rd) << 8 | - imm2 << 6 | - widthminus1; - - Emit32(encoding); -} - - -void Thumb2Assembler::ubfx(Register rd, Register rn, uint32_t lsb, uint32_t width, Condition cond) { - CheckCondition(cond); - CHECK_LE(lsb, 31U); - CHECK(1U <= width && width <= 32U) << width; - uint32_t widthminus1 = width - 1; - uint32_t imm2 = lsb & (B1 | B0); // Bits 0-1 of `lsb`. - uint32_t imm3 = (lsb & (B4 | B3 | B2)) >> 2; // Bits 2-4 of `lsb`. - - uint32_t op = 28U /* 0b11100 */; - int32_t encoding = B31 | B30 | B29 | B28 | B25 | - op << 20 | - static_cast<uint32_t>(rn) << 16 | - imm3 << 12 | - static_cast<uint32_t>(rd) << 8 | - imm2 << 6 | - widthminus1; - - Emit32(encoding); -} - - -void Thumb2Assembler::ldr(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, true, false, false, false, rd, ad); -} - - -void Thumb2Assembler::str(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, false, false, false, false, rd, ad); -} - - -void Thumb2Assembler::ldrb(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, true, true, false, false, rd, ad); -} - - -void Thumb2Assembler::strb(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, false, true, false, false, rd, ad); -} - - -void Thumb2Assembler::ldrh(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, true, false, true, false, rd, ad); -} - - -void Thumb2Assembler::strh(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, false, false, true, false, rd, ad); -} - - -void Thumb2Assembler::ldrsb(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, true, true, false, true, rd, ad); -} - - -void Thumb2Assembler::ldrsh(Register rd, const Address& ad, Condition cond) { - EmitLoadStore(cond, true, false, true, true, rd, ad); -} - - -void Thumb2Assembler::ldrd(Register rd, const Address& ad, Condition cond) { - ldrd(rd, Register(rd + 1), ad, cond); -} - - -void Thumb2Assembler::ldrd(Register rd, Register rd2, const Address& ad, Condition cond) { - CheckCondition(cond); - // Encoding T1. - // This is different from other loads. The encoding is like ARM. - int32_t encoding = B31 | B30 | B29 | B27 | B22 | B20 | - static_cast<int32_t>(rd) << 12 | - static_cast<int32_t>(rd2) << 8 | - ad.encodingThumbLdrdStrd(); - Emit32(encoding); -} - - -void Thumb2Assembler::strd(Register rd, const Address& ad, Condition cond) { - strd(rd, Register(rd + 1), ad, cond); -} - - -void Thumb2Assembler::strd(Register rd, Register rd2, const Address& ad, Condition cond) { - CheckCondition(cond); - // Encoding T1. - // This is different from other loads. The encoding is like ARM. - int32_t encoding = B31 | B30 | B29 | B27 | B22 | - static_cast<int32_t>(rd) << 12 | - static_cast<int32_t>(rd2) << 8 | - ad.encodingThumbLdrdStrd(); - Emit32(encoding); -} - - -void Thumb2Assembler::ldm(BlockAddressMode am, - Register base, - RegList regs, - Condition cond) { - CHECK_NE(regs, 0u); // Do not use ldm if there's nothing to load. - if (IsPowerOfTwo(regs)) { - // Thumb doesn't support one reg in the list. - // Find the register number. - int reg = CTZ(static_cast<uint32_t>(regs)); - CHECK_LT(reg, 16); - CHECK(am == DB_W); // Only writeback is supported. - ldr(static_cast<Register>(reg), Address(base, kRegisterSize, Address::PostIndex), cond); - } else { - EmitMultiMemOp(cond, am, true, base, regs); - } -} - - -void Thumb2Assembler::stm(BlockAddressMode am, - Register base, - RegList regs, - Condition cond) { - CHECK_NE(regs, 0u); // Do not use stm if there's nothing to store. - if (IsPowerOfTwo(regs)) { - // Thumb doesn't support one reg in the list. - // Find the register number. - int reg = CTZ(static_cast<uint32_t>(regs)); - CHECK_LT(reg, 16); - CHECK(am == IA || am == IA_W); - Address::Mode strmode = am == IA ? Address::PreIndex : Address::Offset; - str(static_cast<Register>(reg), Address(base, -kRegisterSize, strmode), cond); - } else { - EmitMultiMemOp(cond, am, false, base, regs); - } -} - - -bool Thumb2Assembler::vmovs(SRegister sd, float s_imm, Condition cond) { - uint32_t imm32 = bit_cast<uint32_t, float>(s_imm); - if (((imm32 & ((1 << 19) - 1)) == 0) && - ((((imm32 >> 25) & ((1 << 6) - 1)) == (1 << 5)) || - (((imm32 >> 25) & ((1 << 6) - 1)) == ((1 << 5) -1)))) { - uint8_t imm8 = ((imm32 >> 31) << 7) | (((imm32 >> 29) & 1) << 6) | - ((imm32 >> 19) & ((1 << 6) -1)); - EmitVFPsss(cond, B23 | B21 | B20 | ((imm8 >> 4)*B16) | (imm8 & 0xf), - sd, S0, S0); - return true; - } - return false; -} - - -bool Thumb2Assembler::vmovd(DRegister dd, double d_imm, Condition cond) { - uint64_t imm64 = bit_cast<uint64_t, double>(d_imm); - if (((imm64 & ((1LL << 48) - 1)) == 0) && - ((((imm64 >> 54) & ((1 << 9) - 1)) == (1 << 8)) || - (((imm64 >> 54) & ((1 << 9) - 1)) == ((1 << 8) -1)))) { - uint8_t imm8 = ((imm64 >> 63) << 7) | (((imm64 >> 61) & 1) << 6) | - ((imm64 >> 48) & ((1 << 6) -1)); - EmitVFPddd(cond, B23 | B21 | B20 | ((imm8 >> 4)*B16) | B8 | (imm8 & 0xf), - dd, D0, D0); - return true; - } - return false; -} - - -void Thumb2Assembler::vmovs(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vmovd(DRegister dd, DRegister dm, Condition cond) { - EmitVFPddd(cond, B23 | B21 | B20 | B6, dd, D0, dm); -} - - -void Thumb2Assembler::vadds(SRegister sd, SRegister sn, SRegister sm, - Condition cond) { - EmitVFPsss(cond, B21 | B20, sd, sn, sm); -} - - -void Thumb2Assembler::vaddd(DRegister dd, DRegister dn, DRegister dm, - Condition cond) { - EmitVFPddd(cond, B21 | B20, dd, dn, dm); -} - - -void Thumb2Assembler::vsubs(SRegister sd, SRegister sn, SRegister sm, - Condition cond) { - EmitVFPsss(cond, B21 | B20 | B6, sd, sn, sm); -} - - -void Thumb2Assembler::vsubd(DRegister dd, DRegister dn, DRegister dm, - Condition cond) { - EmitVFPddd(cond, B21 | B20 | B6, dd, dn, dm); -} - - -void Thumb2Assembler::vmuls(SRegister sd, SRegister sn, SRegister sm, - Condition cond) { - EmitVFPsss(cond, B21, sd, sn, sm); -} - - -void Thumb2Assembler::vmuld(DRegister dd, DRegister dn, DRegister dm, - Condition cond) { - EmitVFPddd(cond, B21, dd, dn, dm); -} - - -void Thumb2Assembler::vmlas(SRegister sd, SRegister sn, SRegister sm, - Condition cond) { - EmitVFPsss(cond, 0, sd, sn, sm); -} - - -void Thumb2Assembler::vmlad(DRegister dd, DRegister dn, DRegister dm, - Condition cond) { - EmitVFPddd(cond, 0, dd, dn, dm); -} - - -void Thumb2Assembler::vmlss(SRegister sd, SRegister sn, SRegister sm, - Condition cond) { - EmitVFPsss(cond, B6, sd, sn, sm); -} - - -void Thumb2Assembler::vmlsd(DRegister dd, DRegister dn, DRegister dm, - Condition cond) { - EmitVFPddd(cond, B6, dd, dn, dm); -} - - -void Thumb2Assembler::vdivs(SRegister sd, SRegister sn, SRegister sm, - Condition cond) { - EmitVFPsss(cond, B23, sd, sn, sm); -} - - -void Thumb2Assembler::vdivd(DRegister dd, DRegister dn, DRegister dm, - Condition cond) { - EmitVFPddd(cond, B23, dd, dn, dm); -} - - -void Thumb2Assembler::vabss(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B7 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vabsd(DRegister dd, DRegister dm, Condition cond) { - EmitVFPddd(cond, B23 | B21 | B20 | B7 | B6, dd, D0, dm); -} - - -void Thumb2Assembler::vnegs(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B16 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vnegd(DRegister dd, DRegister dm, Condition cond) { - EmitVFPddd(cond, B23 | B21 | B20 | B16 | B6, dd, D0, dm); -} - - -void Thumb2Assembler::vsqrts(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B16 | B7 | B6, sd, S0, sm); -} - -void Thumb2Assembler::vsqrtd(DRegister dd, DRegister dm, Condition cond) { - EmitVFPddd(cond, B23 | B21 | B20 | B16 | B7 | B6, dd, D0, dm); -} - - -void Thumb2Assembler::vcvtsd(SRegister sd, DRegister dm, Condition cond) { - EmitVFPsd(cond, B23 | B21 | B20 | B18 | B17 | B16 | B8 | B7 | B6, sd, dm); -} - - -void Thumb2Assembler::vcvtds(DRegister dd, SRegister sm, Condition cond) { - EmitVFPds(cond, B23 | B21 | B20 | B18 | B17 | B16 | B7 | B6, dd, sm); -} - - -void Thumb2Assembler::vcvtis(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B19 | B18 | B16 | B7 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vcvtid(SRegister sd, DRegister dm, Condition cond) { - EmitVFPsd(cond, B23 | B21 | B20 | B19 | B18 | B16 | B8 | B7 | B6, sd, dm); -} - - -void Thumb2Assembler::vcvtsi(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B19 | B7 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vcvtdi(DRegister dd, SRegister sm, Condition cond) { - EmitVFPds(cond, B23 | B21 | B20 | B19 | B8 | B7 | B6, dd, sm); -} - - -void Thumb2Assembler::vcvtus(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B19 | B18 | B7 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vcvtud(SRegister sd, DRegister dm, Condition cond) { - EmitVFPsd(cond, B23 | B21 | B20 | B19 | B18 | B8 | B7 | B6, sd, dm); -} - - -void Thumb2Assembler::vcvtsu(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B19 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vcvtdu(DRegister dd, SRegister sm, Condition cond) { - EmitVFPds(cond, B23 | B21 | B20 | B19 | B8 | B6, dd, sm); -} - - -void Thumb2Assembler::vcmps(SRegister sd, SRegister sm, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B18 | B6, sd, S0, sm); -} - - -void Thumb2Assembler::vcmpd(DRegister dd, DRegister dm, Condition cond) { - EmitVFPddd(cond, B23 | B21 | B20 | B18 | B6, dd, D0, dm); -} - - -void Thumb2Assembler::vcmpsz(SRegister sd, Condition cond) { - EmitVFPsss(cond, B23 | B21 | B20 | B18 | B16 | B6, sd, S0, S0); -} - - -void Thumb2Assembler::vcmpdz(DRegister dd, Condition cond) { - EmitVFPddd(cond, B23 | B21 | B20 | B18 | B16 | B6, dd, D0, D0); -} - -void Thumb2Assembler::b(Label* label, Condition cond) { - DCHECK_EQ(next_condition_, AL); - EmitBranch(cond, label, false, false); -} - - -void Thumb2Assembler::bl(Label* label, Condition cond) { - CheckCondition(cond); - EmitBranch(cond, label, true, false); -} - - -void Thumb2Assembler::blx(Label* label) { - EmitBranch(AL, label, true, true); -} - - -void Thumb2Assembler::MarkExceptionHandler(Label* label) { - EmitDataProcessing(AL, TST, kCcSet, PC, R0, ShifterOperand(0)); - Label l; - b(&l); - EmitBranch(AL, label, false, false); - Bind(&l); -} - - -void Thumb2Assembler::Emit32(int32_t value) { - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - buffer_.Emit<int16_t>(value >> 16); - buffer_.Emit<int16_t>(value & 0xffff); -} - - -void Thumb2Assembler::Emit16(int16_t value) { - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - buffer_.Emit<int16_t>(value); -} - - -bool Thumb2Assembler::Is32BitDataProcessing(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so) { - if (force_32bit_) { - return true; - } - - // Check special case for SP relative ADD and SUB immediate. - if ((opcode == ADD || opcode == SUB) && rn == SP && so.IsImmediate() && set_cc != kCcSet) { - // If the immediate is in range, use 16 bit. - if (rd == SP) { - if (so.GetImmediate() < (1 << 9)) { // 9 bit immediate. - return false; - } - } else if (!IsHighRegister(rd) && opcode == ADD) { - if (so.GetImmediate() < (1 << 10)) { // 10 bit immediate. - return false; - } - } - } - - bool can_contain_high_register = - (opcode == CMP) || - (opcode == MOV && set_cc != kCcSet) || - ((opcode == ADD) && (rn == rd) && set_cc != kCcSet); - - if (IsHighRegister(rd) || IsHighRegister(rn)) { - if (!can_contain_high_register) { - return true; - } - - // There are high register instructions available for this opcode. - // However, there is no actual shift available, neither for ADD nor for MOV (ASR/LSR/LSL/ROR). - if (so.IsShift() && (so.GetShift() == RRX || so.GetImmediate() != 0u)) { - return true; - } - - // The ADD and MOV instructions that work with high registers don't have 16-bit - // immediate variants. - if (so.IsImmediate()) { - return true; - } - } - - if (so.IsRegister() && IsHighRegister(so.GetRegister()) && !can_contain_high_register) { - return true; - } - - bool rn_is_valid = true; - - // Check for single operand instructions and ADD/SUB. - switch (opcode) { - case CMP: - case MOV: - case TST: - case MVN: - rn_is_valid = false; // There is no Rn for these instructions. - break; - case TEQ: - case ORN: - return true; - case ADD: - case SUB: - break; - default: - if (so.IsRegister() && rd != rn) { - return true; - } - } - - if (so.IsImmediate()) { - if (opcode == RSB) { - DCHECK(rn_is_valid); - if (so.GetImmediate() != 0u) { - return true; - } - } else if (rn_is_valid && rn != rd) { - // The only thumb1 instructions with a register and an immediate are ADD and SUB - // with a 3-bit immediate, and RSB with zero immediate. - if (opcode == ADD || opcode == SUB) { - if ((cond == AL) ? set_cc == kCcKeep : set_cc == kCcSet) { - return true; // Cannot match "setflags". - } - if (!IsUint<3>(so.GetImmediate()) && !IsUint<3>(-so.GetImmediate())) { - return true; - } - } else { - return true; - } - } else { - // ADD, SUB, CMP and MOV may be thumb1 only if the immediate is 8 bits. - if (!(opcode == ADD || opcode == SUB || opcode == MOV || opcode == CMP)) { - return true; - } else if (opcode != CMP && ((cond == AL) ? set_cc == kCcKeep : set_cc == kCcSet)) { - return true; // Cannot match "setflags" for ADD, SUB or MOV. - } else { - // For ADD and SUB allow also negative 8-bit immediate as we will emit the oposite opcode. - if (!IsUint<8>(so.GetImmediate()) && - (opcode == MOV || opcode == CMP || !IsUint<8>(-so.GetImmediate()))) { - return true; - } - } - } - } else { - DCHECK(so.IsRegister()); - if (so.IsShift()) { - // Shift operand - check if it is a MOV convertible to a 16-bit shift instruction. - if (opcode != MOV) { - return true; - } - // Check for MOV with an ROR/RRX. There is no 16-bit ROR immediate and no 16-bit RRX. - if (so.GetShift() == ROR || so.GetShift() == RRX) { - return true; - } - // 16-bit shifts set condition codes if and only if outside IT block, - // i.e. if and only if cond == AL. - if ((cond == AL) ? set_cc == kCcKeep : set_cc == kCcSet) { - return true; - } - } else { - // Register operand without shift. - switch (opcode) { - case ADD: - // The 16-bit ADD that cannot contain high registers can set condition codes - // if and only if outside IT block, i.e. if and only if cond == AL. - if (!can_contain_high_register && - ((cond == AL) ? set_cc == kCcKeep : set_cc == kCcSet)) { - return true; - } - break; - case AND: - case BIC: - case EOR: - case ORR: - case MVN: - case ADC: - case SUB: - case SBC: - // These 16-bit opcodes set condition codes if and only if outside IT block, - // i.e. if and only if cond == AL. - if ((cond == AL) ? set_cc == kCcKeep : set_cc == kCcSet) { - return true; - } - break; - case RSB: - case RSC: - // No 16-bit RSB/RSC Rd, Rm, Rn. It would be equivalent to SUB/SBC Rd, Rn, Rm. - return true; - case CMP: - default: - break; - } - } - } - - // The instruction can be encoded in 16 bits. - return false; -} - - -void Thumb2Assembler::Emit32BitDataProcessing(Condition cond ATTRIBUTE_UNUSED, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so) { - uint8_t thumb_opcode = 255U /* 0b11111111 */; - switch (opcode) { - case AND: thumb_opcode = 0U /* 0b0000 */; break; - case EOR: thumb_opcode = 4U /* 0b0100 */; break; - case SUB: thumb_opcode = 13U /* 0b1101 */; break; - case RSB: thumb_opcode = 14U /* 0b1110 */; break; - case ADD: thumb_opcode = 8U /* 0b1000 */; break; - case ADC: thumb_opcode = 10U /* 0b1010 */; break; - case SBC: thumb_opcode = 11U /* 0b1011 */; break; - case RSC: break; - case TST: thumb_opcode = 0U /* 0b0000 */; DCHECK(set_cc == kCcSet); rd = PC; break; - case TEQ: thumb_opcode = 4U /* 0b0100 */; DCHECK(set_cc == kCcSet); rd = PC; break; - case CMP: thumb_opcode = 13U /* 0b1101 */; DCHECK(set_cc == kCcSet); rd = PC; break; - case CMN: thumb_opcode = 8U /* 0b1000 */; DCHECK(set_cc == kCcSet); rd = PC; break; - case ORR: thumb_opcode = 2U /* 0b0010 */; break; - case MOV: thumb_opcode = 2U /* 0b0010 */; rn = PC; break; - case BIC: thumb_opcode = 1U /* 0b0001 */; break; - case MVN: thumb_opcode = 3U /* 0b0011 */; rn = PC; break; - case ORN: thumb_opcode = 3U /* 0b0011 */; break; - default: - break; - } - - if (thumb_opcode == 255U /* 0b11111111 */) { - LOG(FATAL) << "Invalid thumb2 opcode " << opcode; - UNREACHABLE(); - } - - int32_t encoding = 0; - if (so.IsImmediate()) { - // Check special cases. - if ((opcode == SUB || opcode == ADD) && (so.GetImmediate() < (1u << 12)) && - /* Prefer T3 encoding to T4. */ !ShifterOperandCanAlwaysHold(so.GetImmediate())) { - if (set_cc != kCcSet) { - if (opcode == SUB) { - thumb_opcode = 5U; - } else if (opcode == ADD) { - thumb_opcode = 0U; - } - } - uint32_t imm = so.GetImmediate(); - - uint32_t i = (imm >> 11) & 1; - uint32_t imm3 = (imm >> 8) & 7U /* 0b111 */; - uint32_t imm8 = imm & 0xff; - - encoding = B31 | B30 | B29 | B28 | - (set_cc == kCcSet ? B20 : B25) | - thumb_opcode << 21 | - rn << 16 | - rd << 8 | - i << 26 | - imm3 << 12 | - imm8; - } else { - // Modified immediate. - uint32_t imm = ModifiedImmediate(so.encodingThumb()); - if (imm == kInvalidModifiedImmediate) { - LOG(FATAL) << "Immediate value cannot fit in thumb2 modified immediate"; - UNREACHABLE(); - } - encoding = B31 | B30 | B29 | B28 | - thumb_opcode << 21 | - (set_cc == kCcSet ? B20 : 0) | - rn << 16 | - rd << 8 | - imm; - } - } else if (so.IsRegister()) { - // Register (possibly shifted) - encoding = B31 | B30 | B29 | B27 | B25 | - thumb_opcode << 21 | - (set_cc == kCcSet ? B20 : 0) | - rn << 16 | - rd << 8 | - so.encodingThumb(); - } - Emit32(encoding); -} - - -void Thumb2Assembler::Emit16BitDataProcessing(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so) { - if (opcode == ADD || opcode == SUB) { - Emit16BitAddSub(cond, opcode, set_cc, rn, rd, so); - return; - } - uint8_t thumb_opcode = 255U /* 0b11111111 */; - // Thumb1. - uint8_t dp_opcode = 1U /* 0b01 */; - uint8_t opcode_shift = 6; - uint8_t rd_shift = 0; - uint8_t rn_shift = 3; - uint8_t immediate_shift = 0; - bool use_immediate = false; - uint8_t immediate = 0; - - if (opcode == MOV && so.IsRegister() && so.IsShift()) { - // Convert shifted mov operand2 into 16 bit opcodes. - dp_opcode = 0; - opcode_shift = 11; - - use_immediate = true; - immediate = so.GetImmediate(); - immediate_shift = 6; - - rn = so.GetRegister(); - - switch (so.GetShift()) { - case LSL: - DCHECK_LE(immediate, 31u); - thumb_opcode = 0U /* 0b00 */; - break; - case LSR: - DCHECK(1 <= immediate && immediate <= 32); - immediate &= 31; // 32 is encoded as 0. - thumb_opcode = 1U /* 0b01 */; - break; - case ASR: - DCHECK(1 <= immediate && immediate <= 32); - immediate &= 31; // 32 is encoded as 0. - thumb_opcode = 2U /* 0b10 */; - break; - case ROR: // No 16-bit ROR immediate. - case RRX: // No 16-bit RRX. - default: - LOG(FATAL) << "Unexpected shift: " << so.GetShift(); - UNREACHABLE(); - } - } else { - if (so.IsImmediate()) { - use_immediate = true; - immediate = so.GetImmediate(); - } else { - CHECK(!(so.IsRegister() && so.IsShift() && so.GetSecondRegister() != kNoRegister)) - << "No register-shifted register instruction available in thumb"; - // Adjust rn and rd: only two registers will be emitted. - switch (opcode) { - case AND: - case ORR: - case EOR: - case RSB: - case ADC: - case SBC: - case BIC: { - // Sets condition codes if and only if outside IT block, - // check that it complies with set_cc. - DCHECK((cond == AL) ? set_cc != kCcKeep : set_cc != kCcSet); - if (rn == rd) { - rn = so.GetRegister(); - } else { - CHECK_EQ(rd, so.GetRegister()); - } - break; - } - case CMP: - case CMN: { - CHECK_EQ(rd, 0); - rd = rn; - rn = so.GetRegister(); - break; - } - case MVN: { - // Sets condition codes if and only if outside IT block, - // check that it complies with set_cc. - DCHECK((cond == AL) ? set_cc != kCcKeep : set_cc != kCcSet); - CHECK_EQ(rn, 0); - rn = so.GetRegister(); - break; - } - case TST: - case TEQ: { - DCHECK(set_cc == kCcSet); - CHECK_EQ(rn, 0); - rn = so.GetRegister(); - break; - } - default: - break; - } - } - - switch (opcode) { - case AND: thumb_opcode = 0U /* 0b0000 */; break; - case ORR: thumb_opcode = 12U /* 0b1100 */; break; - case EOR: thumb_opcode = 1U /* 0b0001 */; break; - case RSB: thumb_opcode = 9U /* 0b1001 */; break; - case ADC: thumb_opcode = 5U /* 0b0101 */; break; - case SBC: thumb_opcode = 6U /* 0b0110 */; break; - case BIC: thumb_opcode = 14U /* 0b1110 */; break; - case TST: thumb_opcode = 8U /* 0b1000 */; CHECK(!use_immediate); break; - case MVN: thumb_opcode = 15U /* 0b1111 */; CHECK(!use_immediate); break; - case CMP: { - DCHECK(set_cc == kCcSet); - if (use_immediate) { - // T2 encoding. - dp_opcode = 0; - opcode_shift = 11; - thumb_opcode = 5U /* 0b101 */; - rd_shift = 8; - rn_shift = 8; - } else if (IsHighRegister(rd) || IsHighRegister(rn)) { - // Special cmp for high registers. - dp_opcode = 1U /* 0b01 */; - opcode_shift = 7; - // Put the top bit of rd into the bottom bit of the opcode. - thumb_opcode = 10U /* 0b0001010 */ | static_cast<uint32_t>(rd) >> 3; - rd = static_cast<Register>(static_cast<uint32_t>(rd) & 7U /* 0b111 */); - } else { - thumb_opcode = 10U /* 0b1010 */; - } - - break; - } - case CMN: { - CHECK(!use_immediate); - thumb_opcode = 11U /* 0b1011 */; - break; - } - case MOV: - dp_opcode = 0; - if (use_immediate) { - // T2 encoding. - opcode_shift = 11; - thumb_opcode = 4U /* 0b100 */; - rd_shift = 8; - rn_shift = 8; - } else { - rn = so.GetRegister(); - if (set_cc != kCcSet) { - // Special mov for high registers. - dp_opcode = 1U /* 0b01 */; - opcode_shift = 7; - // Put the top bit of rd into the bottom bit of the opcode. - thumb_opcode = 12U /* 0b0001100 */ | static_cast<uint32_t>(rd) >> 3; - rd = static_cast<Register>(static_cast<uint32_t>(rd) & 7U /* 0b111 */); - } else { - DCHECK(!IsHighRegister(rn)); - DCHECK(!IsHighRegister(rd)); - thumb_opcode = 0; - } - } - break; - - case TEQ: - case RSC: - default: - LOG(FATAL) << "Invalid thumb1 opcode " << opcode; - break; - } - } - - if (thumb_opcode == 255U /* 0b11111111 */) { - LOG(FATAL) << "Invalid thumb1 opcode " << opcode; - UNREACHABLE(); - } - - int16_t encoding = dp_opcode << 14 | - (thumb_opcode << opcode_shift) | - rd << rd_shift | - rn << rn_shift | - (use_immediate ? (immediate << immediate_shift) : 0); - - Emit16(encoding); -} - - -// ADD and SUB are complex enough to warrant their own emitter. -void Thumb2Assembler::Emit16BitAddSub(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so) { - uint8_t dp_opcode = 0; - uint8_t opcode_shift = 6; - uint8_t rd_shift = 0; - uint8_t rn_shift = 3; - uint8_t immediate_shift = 0; - bool use_immediate = false; - uint32_t immediate = 0; // Should be at most 10 bits but keep the full immediate for CHECKs. - uint8_t thumb_opcode; - - if (so.IsImmediate()) { - use_immediate = true; - immediate = so.GetImmediate(); - if (!IsUint<10>(immediate)) { - // Flip ADD/SUB. - opcode = (opcode == ADD) ? SUB : ADD; - immediate = -immediate; - DCHECK(IsUint<10>(immediate)); // More stringent checks below. - } - } - - switch (opcode) { - case ADD: - if (so.IsRegister()) { - Register rm = so.GetRegister(); - if (rn == rd && set_cc != kCcSet) { - // Can use T2 encoding (allows 4 bit registers) - dp_opcode = 1U /* 0b01 */; - opcode_shift = 10; - thumb_opcode = 1U /* 0b0001 */; - // Make Rn also contain the top bit of rd. - rn = static_cast<Register>(static_cast<uint32_t>(rm) | - (static_cast<uint32_t>(rd) & 8U /* 0b1000 */) << 1); - rd = static_cast<Register>(static_cast<uint32_t>(rd) & 7U /* 0b111 */); - } else { - // T1. - DCHECK(!IsHighRegister(rd)); - DCHECK(!IsHighRegister(rn)); - DCHECK(!IsHighRegister(rm)); - // Sets condition codes if and only if outside IT block, - // check that it complies with set_cc. - DCHECK((cond == AL) ? set_cc != kCcKeep : set_cc != kCcSet); - opcode_shift = 9; - thumb_opcode = 12U /* 0b01100 */; - immediate = static_cast<uint32_t>(so.GetRegister()); - use_immediate = true; - immediate_shift = 6; - } - } else { - // Immediate. - if (rd == SP && rn == SP) { - // ADD sp, sp, #imm - dp_opcode = 2U /* 0b10 */; - thumb_opcode = 3U /* 0b11 */; - opcode_shift = 12; - CHECK(IsUint<9>(immediate)); - CHECK_ALIGNED(immediate, 4); - - // Remove rd and rn from instruction by orring it with immed and clearing bits. - rn = R0; - rd = R0; - rd_shift = 0; - rn_shift = 0; - immediate >>= 2; - } else if (rd != SP && rn == SP) { - // ADD rd, SP, #imm - dp_opcode = 2U /* 0b10 */; - thumb_opcode = 5U /* 0b101 */; - opcode_shift = 11; - CHECK(IsUint<10>(immediate)); - CHECK_ALIGNED(immediate, 4); - - // Remove rn from instruction. - rn = R0; - rn_shift = 0; - rd_shift = 8; - immediate >>= 2; - } else if (rn != rd) { - // Must use T1. - CHECK(IsUint<3>(immediate)); - opcode_shift = 9; - thumb_opcode = 14U /* 0b01110 */; - immediate_shift = 6; - } else { - // T2 encoding. - CHECK(IsUint<8>(immediate)); - opcode_shift = 11; - thumb_opcode = 6U /* 0b110 */; - rd_shift = 8; - rn_shift = 8; - } - } - break; - - case SUB: - if (so.IsRegister()) { - // T1. - Register rm = so.GetRegister(); - DCHECK(!IsHighRegister(rd)); - DCHECK(!IsHighRegister(rn)); - DCHECK(!IsHighRegister(rm)); - // Sets condition codes if and only if outside IT block, - // check that it complies with set_cc. - DCHECK((cond == AL) ? set_cc != kCcKeep : set_cc != kCcSet); - opcode_shift = 9; - thumb_opcode = 13U /* 0b01101 */; - immediate = static_cast<uint32_t>(rm); - use_immediate = true; - immediate_shift = 6; - } else { - if (rd == SP && rn == SP) { - // SUB sp, sp, #imm - dp_opcode = 2U /* 0b10 */; - thumb_opcode = 0x61 /* 0b1100001 */; - opcode_shift = 7; - CHECK(IsUint<9>(immediate)); - CHECK_ALIGNED(immediate, 4); - - // Remove rd and rn from instruction by orring it with immed and clearing bits. - rn = R0; - rd = R0; - rd_shift = 0; - rn_shift = 0; - immediate >>= 2; - } else if (rn != rd) { - // Must use T1. - CHECK(IsUint<3>(immediate)); - opcode_shift = 9; - thumb_opcode = 15U /* 0b01111 */; - immediate_shift = 6; - } else { - // T2 encoding. - CHECK(IsUint<8>(immediate)); - opcode_shift = 11; - thumb_opcode = 7U /* 0b111 */; - rd_shift = 8; - rn_shift = 8; - } - } - break; - default: - LOG(FATAL) << "This opcode is not an ADD or SUB: " << opcode; - UNREACHABLE(); - } - - int16_t encoding = dp_opcode << 14 | - (thumb_opcode << opcode_shift) | - rd << rd_shift | - rn << rn_shift | - (use_immediate ? (immediate << immediate_shift) : 0); - - Emit16(encoding); -} - - -void Thumb2Assembler::EmitDataProcessing(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so) { - CHECK_NE(rd, kNoRegister); - CheckCondition(cond); - - if (Is32BitDataProcessing(cond, opcode, set_cc, rn, rd, so)) { - Emit32BitDataProcessing(cond, opcode, set_cc, rn, rd, so); - } else { - Emit16BitDataProcessing(cond, opcode, set_cc, rn, rd, so); - } -} - -void Thumb2Assembler::EmitShift(Register rd, - Register rm, - Shift shift, - uint8_t amount, - Condition cond, - SetCc set_cc) { - CHECK_LT(amount, (1 << 5)); - if ((IsHighRegister(rd) || IsHighRegister(rm) || shift == ROR || shift == RRX) || - ((cond == AL) ? set_cc == kCcKeep : set_cc == kCcSet)) { - uint16_t opcode = 0; - switch (shift) { - case LSL: opcode = 0U /* 0b00 */; break; - case LSR: opcode = 1U /* 0b01 */; break; - case ASR: opcode = 2U /* 0b10 */; break; - case ROR: opcode = 3U /* 0b11 */; break; - case RRX: opcode = 3U /* 0b11 */; amount = 0; break; - default: - LOG(FATAL) << "Unsupported thumb2 shift opcode"; - UNREACHABLE(); - } - // 32 bit. - int32_t encoding = B31 | B30 | B29 | B27 | B25 | B22 | - 0xf << 16 | (set_cc == kCcSet ? B20 : 0); - uint32_t imm3 = amount >> 2; - uint32_t imm2 = amount & 3U /* 0b11 */; - encoding |= imm3 << 12 | imm2 << 6 | static_cast<int16_t>(rm) | - static_cast<int16_t>(rd) << 8 | opcode << 4; - Emit32(encoding); - } else { - // 16 bit shift - uint16_t opcode = 0; - switch (shift) { - case LSL: opcode = 0U /* 0b00 */; break; - case LSR: opcode = 1U /* 0b01 */; break; - case ASR: opcode = 2U /* 0b10 */; break; - default: - LOG(FATAL) << "Unsupported thumb2 shift opcode"; - UNREACHABLE(); - } - int16_t encoding = opcode << 11 | amount << 6 | static_cast<int16_t>(rm) << 3 | - static_cast<int16_t>(rd); - Emit16(encoding); - } -} - -void Thumb2Assembler::EmitShift(Register rd, - Register rn, - Shift shift, - Register rm, - Condition cond, - SetCc set_cc) { - CHECK_NE(shift, RRX); - bool must_be_32bit = false; - if (IsHighRegister(rd) || IsHighRegister(rm) || IsHighRegister(rn) || rd != rn || - ((cond == AL) ? set_cc == kCcKeep : set_cc == kCcSet)) { - must_be_32bit = true; - } - - if (must_be_32bit) { - uint16_t opcode = 0; - switch (shift) { - case LSL: opcode = 0U /* 0b00 */; break; - case LSR: opcode = 1U /* 0b01 */; break; - case ASR: opcode = 2U /* 0b10 */; break; - case ROR: opcode = 3U /* 0b11 */; break; - default: - LOG(FATAL) << "Unsupported thumb2 shift opcode"; - UNREACHABLE(); - } - // 32 bit. - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | - 0xf << 12 | (set_cc == kCcSet ? B20 : 0); - encoding |= static_cast<int16_t>(rn) << 16 | static_cast<int16_t>(rm) | - static_cast<int16_t>(rd) << 8 | opcode << 21; - Emit32(encoding); - } else { - uint16_t opcode = 0; - switch (shift) { - case LSL: opcode = 2U /* 0b0010 */; break; - case LSR: opcode = 3U /* 0b0011 */; break; - case ASR: opcode = 4U /* 0b0100 */; break; - case ROR: opcode = 7U /* 0b0111 */; break; - default: - LOG(FATAL) << "Unsupported thumb2 shift opcode"; - UNREACHABLE(); - } - int16_t encoding = B14 | opcode << 6 | static_cast<int16_t>(rm) << 3 | - static_cast<int16_t>(rd); - Emit16(encoding); - } -} - -inline size_t Thumb2Assembler::Fixup::SizeInBytes(Size size) { - switch (size) { - case kBranch16Bit: - return 2u; - case kBranch32Bit: - return 4u; - - case kCbxz16Bit: - return 2u; - case kCbxz32Bit: - return 4u; - case kCbxz48Bit: - return 6u; - - case kCodeAddr4KiB: - return 4u; - - case kLiteral1KiB: - return 2u; - case kLiteral4KiB: - return 4u; - case kLiteral64KiB: - return 8u; - case kLiteral1MiB: - return 10u; - case kLiteralFar: - return 14u; - - case kLiteralAddr1KiB: - return 2u; - case kLiteralAddr4KiB: - return 4u; - case kLiteralAddr64KiB: - return 6u; - case kLiteralAddrFar: - return 10u; - - case kLongOrFPLiteral1KiB: - return 4u; - case kLongOrFPLiteral64KiB: - return 10u; - case kLongOrFPLiteralFar: - return 14u; - } - LOG(FATAL) << "Unexpected size: " << static_cast<int>(size); - UNREACHABLE(); -} - -inline uint32_t Thumb2Assembler::Fixup::GetOriginalSizeInBytes() const { - return SizeInBytes(original_size_); -} - -inline uint32_t Thumb2Assembler::Fixup::GetSizeInBytes() const { - return SizeInBytes(size_); -} - -inline size_t Thumb2Assembler::Fixup::LiteralPoolPaddingSize(uint32_t current_code_size) { - // The code size must be a multiple of 2. - DCHECK_ALIGNED(current_code_size, 2); - // If it isn't a multiple of 4, we need to add a 2-byte padding before the literal pool. - return current_code_size & 2; -} - -inline int32_t Thumb2Assembler::Fixup::GetOffset(uint32_t current_code_size) const { - static constexpr int32_t int32_min = std::numeric_limits<int32_t>::min(); - static constexpr int32_t int32_max = std::numeric_limits<int32_t>::max(); - DCHECK_LE(target_, static_cast<uint32_t>(int32_max)); - DCHECK_LE(location_, static_cast<uint32_t>(int32_max)); - DCHECK_LE(adjustment_, static_cast<uint32_t>(int32_max)); - int32_t diff = static_cast<int32_t>(target_) - static_cast<int32_t>(location_); - if (target_ > location_) { - DCHECK_LE(adjustment_, static_cast<uint32_t>(int32_max - diff)); - diff += static_cast<int32_t>(adjustment_); - } else { - DCHECK_LE(int32_min + static_cast<int32_t>(adjustment_), diff); - diff -= static_cast<int32_t>(adjustment_); - } - // The default PC adjustment for Thumb2 is 4 bytes. - DCHECK_GE(diff, int32_min + 4); - diff -= 4; - // Add additional adjustment for instructions preceding the PC usage, padding - // before the literal pool and rounding down the PC for literal loads. - switch (GetSize()) { - case kBranch16Bit: - case kBranch32Bit: - break; - - case kCbxz16Bit: - break; - case kCbxz32Bit: - case kCbxz48Bit: - DCHECK_GE(diff, int32_min + 2); - diff -= 2; // Extra CMP Rn, #0, 16-bit. - break; - - case kCodeAddr4KiB: - // The ADR instruction rounds down the PC+4 to a multiple of 4, so if the PC - // isn't a multiple of 2, we need to adjust. - DCHECK_ALIGNED(diff, 2); - diff += location_ & 2; - // Add the Thumb mode bit. - diff += 1; - break; - - case kLiteral1KiB: - case kLiteral4KiB: - case kLongOrFPLiteral1KiB: - case kLiteralAddr1KiB: - case kLiteralAddr4KiB: - DCHECK(diff >= 0 || (GetSize() == kLiteral1KiB && diff == -2)); - diff += LiteralPoolPaddingSize(current_code_size); - // Load literal instructions round down the PC+4 to a multiple of 4, so if the PC - // isn't a multiple of 2, we need to adjust. Since we already adjusted for the target - // being aligned, current PC alignment can be inferred from diff. - DCHECK_ALIGNED(diff, 2); - diff = diff + (diff & 2); - DCHECK_GE(diff, 0); - break; - case kLiteral64KiB: - case kLiteral1MiB: - case kLongOrFPLiteral64KiB: - case kLiteralAddr64KiB: - DCHECK_GE(diff, 4); // The target must be at least 4 bytes after the ADD rX, PC. - diff -= 4; // One extra 32-bit MOV. - diff += LiteralPoolPaddingSize(current_code_size); - break; - case kLiteralFar: - case kLongOrFPLiteralFar: - case kLiteralAddrFar: - DCHECK_GE(diff, 8); // The target must be at least 4 bytes after the ADD rX, PC. - diff -= 8; // Extra MOVW+MOVT; both 32-bit. - diff += LiteralPoolPaddingSize(current_code_size); - break; - } - return diff; -} - -inline size_t Thumb2Assembler::Fixup::IncreaseSize(Size new_size) { - DCHECK_NE(target_, kUnresolved); - Size old_size = size_; - size_ = new_size; - DCHECK_GT(SizeInBytes(new_size), SizeInBytes(old_size)); - size_t adjustment = SizeInBytes(new_size) - SizeInBytes(old_size); - if (target_ > location_) { - adjustment_ += adjustment; - } - return adjustment; -} - -bool Thumb2Assembler::Fixup::IsCandidateForEmitEarly() const { - DCHECK(size_ == original_size_); - if (target_ == kUnresolved) { - return false; - } - // GetOffset() does not depend on current_code_size for branches, only for literals. - constexpr uint32_t current_code_size = 0u; - switch (GetSize()) { - case kBranch16Bit: - return IsInt(cond_ != AL ? 9 : 12, GetOffset(current_code_size)); - case kBranch32Bit: - // We don't support conditional branches beyond +-1MiB - // or unconditional branches beyond +-16MiB. - return true; - - case kCbxz16Bit: - return IsUint<7>(GetOffset(current_code_size)); - case kCbxz32Bit: - return IsInt<9>(GetOffset(current_code_size)); - case kCbxz48Bit: - // We don't support conditional branches beyond +-1MiB. - return true; - - case kCodeAddr4KiB: - // ADR uses the aligned PC and as such the offset cannot be calculated early. - return false; - - case kLiteral1KiB: - case kLiteral4KiB: - case kLiteral64KiB: - case kLiteral1MiB: - case kLiteralFar: - case kLiteralAddr1KiB: - case kLiteralAddr4KiB: - case kLiteralAddr64KiB: - case kLiteralAddrFar: - case kLongOrFPLiteral1KiB: - case kLongOrFPLiteral64KiB: - case kLongOrFPLiteralFar: - return false; - } -} - -uint32_t Thumb2Assembler::Fixup::AdjustSizeIfNeeded(uint32_t current_code_size) { - uint32_t old_code_size = current_code_size; - switch (GetSize()) { - case kBranch16Bit: - if (IsInt(cond_ != AL ? 9 : 12, GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kBranch32Bit); - FALLTHROUGH_INTENDED; - case kBranch32Bit: - // We don't support conditional branches beyond +-1MiB - // or unconditional branches beyond +-16MiB. - break; - - case kCbxz16Bit: - if (IsUint<7>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kCbxz32Bit); - FALLTHROUGH_INTENDED; - case kCbxz32Bit: - if (IsInt<9>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kCbxz48Bit); - FALLTHROUGH_INTENDED; - case kCbxz48Bit: - // We don't support conditional branches beyond +-1MiB. - break; - - case kCodeAddr4KiB: - // We don't support Code address ADR beyond +4KiB. - break; - - case kLiteral1KiB: - DCHECK(!IsHighRegister(rn_)); - if (IsUint<10>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLiteral4KiB); - FALLTHROUGH_INTENDED; - case kLiteral4KiB: - if (IsUint<12>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLiteral64KiB); - FALLTHROUGH_INTENDED; - case kLiteral64KiB: - // Can't handle high register which we can encounter by fall-through from kLiteral4KiB. - if (!IsHighRegister(rn_) && IsUint<16>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLiteral1MiB); - FALLTHROUGH_INTENDED; - case kLiteral1MiB: - if (IsUint<20>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLiteralFar); - FALLTHROUGH_INTENDED; - case kLiteralFar: - // This encoding can reach any target. - break; - - case kLiteralAddr1KiB: - DCHECK(!IsHighRegister(rn_)); - if (IsUint<10>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLiteralAddr4KiB); - FALLTHROUGH_INTENDED; - case kLiteralAddr4KiB: - if (IsUint<12>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLiteralAddr64KiB); - FALLTHROUGH_INTENDED; - case kLiteralAddr64KiB: - if (IsUint<16>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLiteralAddrFar); - FALLTHROUGH_INTENDED; - case kLiteralAddrFar: - // This encoding can reach any target. - break; - - case kLongOrFPLiteral1KiB: - if (IsUint<10>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLongOrFPLiteral64KiB); - FALLTHROUGH_INTENDED; - case kLongOrFPLiteral64KiB: - if (IsUint<16>(GetOffset(current_code_size))) { - break; - } - current_code_size += IncreaseSize(kLongOrFPLiteralFar); - FALLTHROUGH_INTENDED; - case kLongOrFPLiteralFar: - // This encoding can reach any target. - break; - } - return current_code_size - old_code_size; -} - -void Thumb2Assembler::Fixup::Emit(uint32_t emit_location, - AssemblerBuffer* buffer, - uint32_t code_size) const { - switch (GetSize()) { - case kBranch16Bit: { - DCHECK(type_ == kUnconditional || type_ == kConditional); - DCHECK_EQ(type_ == kConditional, cond_ != AL); - int16_t encoding = BEncoding16(GetOffset(code_size), cond_); - buffer->Store<int16_t>(emit_location, encoding); - break; - } - case kBranch32Bit: { - DCHECK(type_ == kConditional || type_ == kUnconditional || - type_ == kUnconditionalLink || type_ == kUnconditionalLinkX); - DCHECK_EQ(type_ == kConditional, cond_ != AL); - int32_t encoding = BEncoding32(GetOffset(code_size), cond_); - if (type_ == kUnconditionalLink) { - DCHECK_NE(encoding & B12, 0); - encoding |= B14; - } else if (type_ == kUnconditionalLinkX) { - DCHECK_NE(encoding & B12, 0); - encoding ^= B14 | B12; - } - buffer->Store<int16_t>(emit_location, encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(encoding & 0xffff)); - break; - } - - case kCbxz16Bit: { - DCHECK(type_ == kCompareAndBranchXZero); - int16_t encoding = CbxzEncoding16(rn_, GetOffset(code_size), cond_); - buffer->Store<int16_t>(emit_location, encoding); - break; - } - case kCbxz32Bit: { - DCHECK(type_ == kCompareAndBranchXZero); - DCHECK(cond_ == EQ || cond_ == NE); - int16_t cmp_encoding = CmpRnImm8Encoding16(rn_, 0); - int16_t b_encoding = BEncoding16(GetOffset(code_size), cond_); - buffer->Store<int16_t>(emit_location, cmp_encoding); - buffer->Store<int16_t>(emit_location + 2, b_encoding); - break; - } - case kCbxz48Bit: { - DCHECK(type_ == kCompareAndBranchXZero); - DCHECK(cond_ == EQ || cond_ == NE); - int16_t cmp_encoding = CmpRnImm8Encoding16(rn_, 0); - int32_t b_encoding = BEncoding32(GetOffset(code_size), cond_); - buffer->Store<int16_t>(emit_location, cmp_encoding); - buffer->Store<int16_t>(emit_location + 2u, b_encoding >> 16); - buffer->Store<int16_t>(emit_location + 4u, static_cast<int16_t>(b_encoding & 0xffff)); - break; - } - - case kCodeAddr4KiB: { - DCHECK(type_ == kLoadCodeAddr); - int32_t encoding = AdrEncoding32(rn_, GetOffset(code_size)); - buffer->Store<int16_t>(emit_location, encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(encoding & 0xffff)); - break; - } - - case kLiteral1KiB: { - DCHECK(type_ == kLoadLiteralNarrow); - int16_t encoding = LdrLitEncoding16(rn_, GetOffset(code_size)); - buffer->Store<int16_t>(emit_location, encoding); - break; - } - case kLiteral4KiB: { - DCHECK(type_ == kLoadLiteralNarrow); - // GetOffset() uses PC+4 but load literal uses AlignDown(PC+4, 4). Adjust offset accordingly. - int32_t encoding = LdrLitEncoding32(rn_, GetOffset(code_size)); - buffer->Store<int16_t>(emit_location, encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(encoding & 0xffff)); - break; - } - case kLiteral64KiB: { - DCHECK(type_ == kLoadLiteralNarrow); - int32_t mov_encoding = MovwEncoding32(rn_, GetOffset(code_size)); - int16_t add_pc_encoding = AddRdnRmEncoding16(rn_, PC); - int16_t ldr_encoding = LdrRtRnImm5Encoding16(rn_, rn_, 0); - buffer->Store<int16_t>(location_, mov_encoding >> 16); - buffer->Store<int16_t>(location_ + 2u, static_cast<int16_t>(mov_encoding & 0xffff)); - buffer->Store<int16_t>(location_ + 4u, add_pc_encoding); - buffer->Store<int16_t>(location_ + 6u, ldr_encoding); - break; - } - case kLiteral1MiB: { - DCHECK(type_ == kLoadLiteralNarrow); - int32_t offset = GetOffset(code_size); - int32_t mov_encoding = MovModImmEncoding32(rn_, offset & ~0xfff); - int16_t add_pc_encoding = AddRdnRmEncoding16(rn_, PC); - int32_t ldr_encoding = LdrRtRnImm12Encoding(rn_, rn_, offset & 0xfff); - buffer->Store<int16_t>(emit_location, mov_encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(mov_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 4u, add_pc_encoding); - buffer->Store<int16_t>(emit_location + 6u, ldr_encoding >> 16); - buffer->Store<int16_t>(emit_location + 8u, static_cast<int16_t>(ldr_encoding & 0xffff)); - break; - } - case kLiteralFar: { - DCHECK(type_ == kLoadLiteralNarrow); - int32_t offset = GetOffset(code_size); - int32_t movw_encoding = MovwEncoding32(rn_, offset & 0xffff); - int32_t movt_encoding = MovtEncoding32(rn_, offset & ~0xffff); - int16_t add_pc_encoding = AddRdnRmEncoding16(rn_, PC); - int32_t ldr_encoding = LdrRtRnImm12Encoding(rn_, rn_, 0); - buffer->Store<int16_t>(emit_location, movw_encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(movw_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 4u, movt_encoding >> 16); - buffer->Store<int16_t>(emit_location + 6u, static_cast<int16_t>(movt_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 8u, add_pc_encoding); - buffer->Store<int16_t>(emit_location + 10u, ldr_encoding >> 16); - buffer->Store<int16_t>(emit_location + 12u, static_cast<int16_t>(ldr_encoding & 0xffff)); - break; - } - - case kLiteralAddr1KiB: { - DCHECK(type_ == kLoadLiteralAddr); - int16_t encoding = AdrEncoding16(rn_, GetOffset(code_size)); - buffer->Store<int16_t>(emit_location, encoding); - break; - } - case kLiteralAddr4KiB: { - DCHECK(type_ == kLoadLiteralAddr); - int32_t encoding = AdrEncoding32(rn_, GetOffset(code_size)); - buffer->Store<int16_t>(emit_location, encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(encoding & 0xffff)); - break; - } - case kLiteralAddr64KiB: { - DCHECK(type_ == kLoadLiteralAddr); - int32_t mov_encoding = MovwEncoding32(rn_, GetOffset(code_size)); - int16_t add_pc_encoding = AddRdnRmEncoding16(rn_, PC); - buffer->Store<int16_t>(emit_location, mov_encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(mov_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 4u, add_pc_encoding); - break; - } - case kLiteralAddrFar: { - DCHECK(type_ == kLoadLiteralAddr); - int32_t offset = GetOffset(code_size); - int32_t movw_encoding = MovwEncoding32(rn_, offset & 0xffff); - int32_t movt_encoding = MovtEncoding32(rn_, offset & ~0xffff); - int16_t add_pc_encoding = AddRdnRmEncoding16(rn_, PC); - buffer->Store<int16_t>(emit_location, movw_encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(movw_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 4u, movt_encoding >> 16); - buffer->Store<int16_t>(emit_location + 6u, static_cast<int16_t>(movt_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 8u, add_pc_encoding); - break; - } - - case kLongOrFPLiteral1KiB: { - int32_t encoding = LoadWideOrFpEncoding(PC, GetOffset(code_size)); // DCHECKs type_. - buffer->Store<int16_t>(emit_location, encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(encoding & 0xffff)); - break; - } - case kLongOrFPLiteral64KiB: { - int32_t mov_encoding = MovwEncoding32(IP, GetOffset(code_size)); - int16_t add_pc_encoding = AddRdnRmEncoding16(IP, PC); - int32_t ldr_encoding = LoadWideOrFpEncoding(IP, 0u); // DCHECKs type_. - buffer->Store<int16_t>(emit_location, mov_encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(mov_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 4u, add_pc_encoding); - buffer->Store<int16_t>(emit_location + 6u, ldr_encoding >> 16); - buffer->Store<int16_t>(emit_location + 8u, static_cast<int16_t>(ldr_encoding & 0xffff)); - break; - } - case kLongOrFPLiteralFar: { - int32_t offset = GetOffset(code_size); - int32_t movw_encoding = MovwEncoding32(IP, offset & 0xffff); - int32_t movt_encoding = MovtEncoding32(IP, offset & ~0xffff); - int16_t add_pc_encoding = AddRdnRmEncoding16(IP, PC); - int32_t ldr_encoding = LoadWideOrFpEncoding(IP, 0); // DCHECKs type_. - buffer->Store<int16_t>(emit_location, movw_encoding >> 16); - buffer->Store<int16_t>(emit_location + 2u, static_cast<int16_t>(movw_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 4u, movt_encoding >> 16); - buffer->Store<int16_t>(emit_location + 6u, static_cast<int16_t>(movt_encoding & 0xffff)); - buffer->Store<int16_t>(emit_location + 8u, add_pc_encoding); - buffer->Store<int16_t>(emit_location + 10u, ldr_encoding >> 16); - buffer->Store<int16_t>(emit_location + 12u, static_cast<int16_t>(ldr_encoding & 0xffff)); - break; - } - } -} - -uint16_t Thumb2Assembler::EmitCompareAndBranch(Register rn, uint16_t prev, bool n) { - CHECK(IsLowRegister(rn)); - uint32_t location = buffer_.Size(); - - // This is always unresolved as it must be a forward branch. - Emit16(prev); // Previous link. - return AddFixup(Fixup::CompareAndBranch(location, rn, n ? NE : EQ)); -} - - -// NOTE: this only support immediate offsets, not [rx,ry]. -// TODO: support [rx,ry] instructions. -void Thumb2Assembler::EmitLoadStore(Condition cond, - bool load, - bool byte, - bool half, - bool is_signed, - Register rd, - const Address& ad) { - CHECK_NE(rd, kNoRegister); - CheckCondition(cond); - bool must_be_32bit = force_32bit_; - if (IsHighRegister(rd)) { - must_be_32bit = true; - } - - Register rn = ad.GetRegister(); - if (IsHighRegister(rn) && (byte || half || (rn != SP && rn != PC))) { - must_be_32bit = true; - } - - if (is_signed || ad.GetOffset() < 0 || ad.GetMode() != Address::Offset) { - must_be_32bit = true; - } - - if (ad.IsImmediate()) { - // Immediate offset - int32_t offset = ad.GetOffset(); - - if (byte) { - // 5 bit offset, no shift. - if ((offset & ~0x1f) != 0) { - must_be_32bit = true; - } - } else if (half) { - // 5 bit offset, shifted by 1. - if ((offset & ~(0x1f << 1)) != 0) { - must_be_32bit = true; - } - } else if (rn == SP || rn == PC) { - // The 16 bit SP/PC relative instruction can only have an (imm8 << 2) offset. - if ((offset & ~(0xff << 2)) != 0) { - must_be_32bit = true; - } - } else { - // 5 bit offset, shifted by 2. - if ((offset & ~(0x1f << 2)) != 0) { - must_be_32bit = true; - } - } - - if (must_be_32bit) { - int32_t encoding = B31 | B30 | B29 | B28 | B27 | - (load ? B20 : 0) | - (is_signed ? B24 : 0) | - static_cast<uint32_t>(rd) << 12 | - ad.encodingThumb(true) | - (byte ? 0 : half ? B21 : B22); - Emit32(encoding); - } else { - // 16 bit thumb1. - uint8_t opA = 0; - bool sp_or_pc_relative = false; - - if (byte) { - opA = 7U /* 0b0111 */; - } else if (half) { - opA = 8U /* 0b1000 */; - } else { - if (rn == SP) { - opA = 9U /* 0b1001 */; - sp_or_pc_relative = true; - } else if (rn == PC) { - opA = 4U; - sp_or_pc_relative = true; - } else { - opA = 6U /* 0b0110 */; - } - } - int16_t encoding = opA << 12 | - (load ? B11 : 0); - - CHECK_GE(offset, 0); - if (sp_or_pc_relative) { - // SP relative, 10 bit offset. - CHECK_LT(offset, (1 << 10)); - CHECK_ALIGNED(offset, 4); - encoding |= rd << 8 | offset >> 2; - } else { - // No SP relative. The offset is shifted right depending on - // the size of the load/store. - encoding |= static_cast<uint32_t>(rd); - - if (byte) { - // 5 bit offset, no shift. - CHECK_LT(offset, (1 << 5)); - } else if (half) { - // 6 bit offset, shifted by 1. - CHECK_LT(offset, (1 << 6)); - CHECK_ALIGNED(offset, 2); - offset >>= 1; - } else { - // 7 bit offset, shifted by 2. - CHECK_LT(offset, (1 << 7)); - CHECK_ALIGNED(offset, 4); - offset >>= 2; - } - encoding |= rn << 3 | offset << 6; - } - - Emit16(encoding); - } - } else { - // Register shift. - CHECK_NE(ad.GetRegister(), PC); - if (ad.GetShiftCount() != 0) { - // If there is a shift count this must be 32 bit. - must_be_32bit = true; - } else if (IsHighRegister(ad.GetRegisterOffset())) { - must_be_32bit = true; - } - - if (must_be_32bit) { - int32_t encoding = 0x1f << 27 | (load ? B20 : 0) | static_cast<uint32_t>(rd) << 12 | - ad.encodingThumb(true); - if (half) { - encoding |= B21; - } else if (!byte) { - encoding |= B22; - } - if (load && is_signed && (byte || half)) { - encoding |= B24; - } - Emit32(encoding); - } else { - // 16 bit register offset. - int32_t encoding = B14 | B12 | (load ? B11 : 0) | static_cast<uint32_t>(rd) | - ad.encodingThumb(false); - if (byte) { - encoding |= B10; - } else if (half) { - encoding |= B9; - } - Emit16(encoding); - } - } -} - - -void Thumb2Assembler::EmitMultiMemOp(Condition cond, - BlockAddressMode bam, - bool load, - Register base, - RegList regs) { - CHECK_NE(base, kNoRegister); - CheckCondition(cond); - bool must_be_32bit = force_32bit_; - - if (!must_be_32bit && base == SP && bam == (load ? IA_W : DB_W) && - (regs & 0xff00 & ~(1 << (load ? PC : LR))) == 0) { - // Use 16-bit PUSH/POP. - int16_t encoding = B15 | B13 | B12 | (load ? B11 : 0) | B10 | - ((regs & (1 << (load ? PC : LR))) != 0 ? B8 : 0) | (regs & 0x00ff); - Emit16(encoding); - return; - } - - if ((regs & 0xff00) != 0) { - must_be_32bit = true; - } - - bool w_bit = bam == IA_W || bam == DB_W || bam == DA_W || bam == IB_W; - // 16 bit always uses writeback. - if (!w_bit) { - must_be_32bit = true; - } - - if (must_be_32bit) { - uint32_t op = 0; - switch (bam) { - case IA: - case IA_W: - op = 1U /* 0b01 */; - break; - case DB: - case DB_W: - op = 2U /* 0b10 */; - break; - case DA: - case IB: - case DA_W: - case IB_W: - LOG(FATAL) << "LDM/STM mode not supported on thumb: " << bam; - UNREACHABLE(); - } - if (load) { - // Cannot have SP in the list. - CHECK_EQ((regs & (1 << SP)), 0); - } else { - // Cannot have PC or SP in the list. - CHECK_EQ((regs & (1 << PC | 1 << SP)), 0); - } - int32_t encoding = B31 | B30 | B29 | B27 | - (op << 23) | - (load ? B20 : 0) | - base << 16 | - regs | - (w_bit << 21); - Emit32(encoding); - } else { - int16_t encoding = B15 | B14 | - (load ? B11 : 0) | - base << 8 | - regs; - Emit16(encoding); - } -} - -void Thumb2Assembler::EmitBranch(Condition cond, Label* label, bool link, bool x) { - bool use32bit = IsForced32Bit() || !CanRelocateBranches(); - uint32_t pc = buffer_.Size(); - Fixup::Type branch_type; - if (cond == AL) { - if (link) { - use32bit = true; - if (x) { - branch_type = Fixup::kUnconditionalLinkX; // BLX. - } else { - branch_type = Fixup::kUnconditionalLink; // BX. - } - } else { - branch_type = Fixup::kUnconditional; // B. - // The T2 encoding offset is `SignExtend(imm11:'0', 32)` and there is a PC adjustment of 4. - static constexpr size_t kMaxT2BackwardDistance = (1u << 11) - 4u; - if (!use32bit && label->IsBound() && pc - label->Position() > kMaxT2BackwardDistance) { - use32bit = true; - } - } - } else { - branch_type = Fixup::kConditional; // B<cond>. - // The T1 encoding offset is `SignExtend(imm8:'0', 32)` and there is a PC adjustment of 4. - static constexpr size_t kMaxT1BackwardDistance = (1u << 8) - 4u; - if (!use32bit && label->IsBound() && pc - label->Position() > kMaxT1BackwardDistance) { - use32bit = true; - } - } - - Fixup::Size size = use32bit ? Fixup::kBranch32Bit : Fixup::kBranch16Bit; - FixupId branch_id = AddFixup(Fixup::Branch(pc, branch_type, size, cond)); - - if (label->IsBound()) { - // The branch is to a bound label which means that it's a backwards branch. - GetFixup(branch_id)->Resolve(label->Position()); - Emit16(0); - } else { - // Branch target is an unbound label. Add it to a singly-linked list maintained within - // the code with the label serving as the head. - Emit16(static_cast<uint16_t>(label->position_)); - label->LinkTo(branch_id); - } - - if (use32bit) { - Emit16(0); - } - DCHECK_EQ(buffer_.Size() - pc, GetFixup(branch_id)->GetSizeInBytes()); -} - - -void Thumb2Assembler::Emit32Miscellaneous(uint8_t op1, - uint8_t op2, - uint32_t rest_encoding) { - int32_t encoding = B31 | B30 | B29 | B28 | B27 | B25 | B23 | - op1 << 20 | - 0xf << 12 | - B7 | - op2 << 4 | - rest_encoding; - Emit32(encoding); -} - - -void Thumb2Assembler::Emit16Miscellaneous(uint32_t rest_encoding) { - int16_t encoding = B15 | B13 | B12 | - rest_encoding; - Emit16(encoding); -} - -void Thumb2Assembler::clz(Register rd, Register rm, Condition cond) { - CHECK_NE(rd, kNoRegister); - CHECK_NE(rm, kNoRegister); - CheckCondition(cond); - CHECK_NE(rd, PC); - CHECK_NE(rm, PC); - int32_t encoding = - static_cast<uint32_t>(rm) << 16 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(rm); - Emit32Miscellaneous(0b11, 0b00, encoding); -} - - -void Thumb2Assembler::movw(Register rd, uint16_t imm16, Condition cond) { - CheckCondition(cond); - // Always 32 bits, encoding T3. (Other encondings are called MOV, not MOVW.) - uint32_t imm4 = (imm16 >> 12) & 15U /* 0b1111 */; - uint32_t i = (imm16 >> 11) & 1U /* 0b1 */; - uint32_t imm3 = (imm16 >> 8) & 7U /* 0b111 */; - uint32_t imm8 = imm16 & 0xff; - int32_t encoding = B31 | B30 | B29 | B28 | - B25 | B22 | - static_cast<uint32_t>(rd) << 8 | - i << 26 | - imm4 << 16 | - imm3 << 12 | - imm8; - Emit32(encoding); -} - - -void Thumb2Assembler::movt(Register rd, uint16_t imm16, Condition cond) { - CheckCondition(cond); - // Always 32 bits. - uint32_t imm4 = (imm16 >> 12) & 15U /* 0b1111 */; - uint32_t i = (imm16 >> 11) & 1U /* 0b1 */; - uint32_t imm3 = (imm16 >> 8) & 7U /* 0b111 */; - uint32_t imm8 = imm16 & 0xff; - int32_t encoding = B31 | B30 | B29 | B28 | - B25 | B23 | B22 | - static_cast<uint32_t>(rd) << 8 | - i << 26 | - imm4 << 16 | - imm3 << 12 | - imm8; - Emit32(encoding); -} - - -void Thumb2Assembler::rbit(Register rd, Register rm, Condition cond) { - CHECK_NE(rd, kNoRegister); - CHECK_NE(rm, kNoRegister); - CheckCondition(cond); - CHECK_NE(rd, PC); - CHECK_NE(rm, PC); - CHECK_NE(rd, SP); - CHECK_NE(rm, SP); - int32_t encoding = - static_cast<uint32_t>(rm) << 16 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(rm); - - Emit32Miscellaneous(0b01, 0b10, encoding); -} - - -void Thumb2Assembler::EmitReverseBytes(Register rd, Register rm, - uint32_t op) { - CHECK_NE(rd, kNoRegister); - CHECK_NE(rm, kNoRegister); - CHECK_NE(rd, PC); - CHECK_NE(rm, PC); - CHECK_NE(rd, SP); - CHECK_NE(rm, SP); - - if (!IsHighRegister(rd) && !IsHighRegister(rm) && !force_32bit_) { - uint16_t t1_op = B11 | B9 | (op << 6); - int16_t encoding = t1_op | - static_cast<uint16_t>(rm) << 3 | - static_cast<uint16_t>(rd); - Emit16Miscellaneous(encoding); - } else { - int32_t encoding = - static_cast<uint32_t>(rm) << 16 | - static_cast<uint32_t>(rd) << 8 | - static_cast<uint32_t>(rm); - Emit32Miscellaneous(0b01, op, encoding); - } -} - - -void Thumb2Assembler::rev(Register rd, Register rm, Condition cond) { - CheckCondition(cond); - EmitReverseBytes(rd, rm, 0b00); -} - - -void Thumb2Assembler::rev16(Register rd, Register rm, Condition cond) { - CheckCondition(cond); - EmitReverseBytes(rd, rm, 0b01); -} - - -void Thumb2Assembler::revsh(Register rd, Register rm, Condition cond) { - CheckCondition(cond); - EmitReverseBytes(rd, rm, 0b11); -} - - -void Thumb2Assembler::ldrex(Register rt, Register rn, uint16_t imm, Condition cond) { - CHECK_NE(rn, kNoRegister); - CHECK_NE(rt, kNoRegister); - CheckCondition(cond); - CHECK_LT(imm, (1u << 10)); - - int32_t encoding = B31 | B30 | B29 | B27 | B22 | B20 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rt) << 12 | - 0xf << 8 | - imm >> 2; - Emit32(encoding); -} - - -void Thumb2Assembler::ldrex(Register rt, Register rn, Condition cond) { - ldrex(rt, rn, 0, cond); -} - - -void Thumb2Assembler::strex(Register rd, - Register rt, - Register rn, - uint16_t imm, - Condition cond) { - CHECK_NE(rn, kNoRegister); - CHECK_NE(rd, kNoRegister); - CHECK_NE(rt, kNoRegister); - CheckCondition(cond); - CHECK_LT(imm, (1u << 10)); - - int32_t encoding = B31 | B30 | B29 | B27 | B22 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rt) << 12 | - static_cast<uint32_t>(rd) << 8 | - imm >> 2; - Emit32(encoding); -} - - -void Thumb2Assembler::ldrexd(Register rt, Register rt2, Register rn, Condition cond) { - CHECK_NE(rn, kNoRegister); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt2, kNoRegister); - CHECK_NE(rt, rt2); - CheckCondition(cond); - - int32_t encoding = B31 | B30 | B29 | B27 | B23 | B22 | B20 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rt) << 12 | - static_cast<uint32_t>(rt2) << 8 | - B6 | B5 | B4 | B3 | B2 | B1 | B0; - Emit32(encoding); -} - - -void Thumb2Assembler::strex(Register rd, - Register rt, - Register rn, - Condition cond) { - strex(rd, rt, rn, 0, cond); -} - - -void Thumb2Assembler::strexd(Register rd, Register rt, Register rt2, Register rn, Condition cond) { - CHECK_NE(rd, kNoRegister); - CHECK_NE(rn, kNoRegister); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt2, kNoRegister); - CHECK_NE(rt, rt2); - CHECK_NE(rd, rt); - CHECK_NE(rd, rt2); - CheckCondition(cond); - - int32_t encoding = B31 | B30 | B29 | B27 | B23 | B22 | - static_cast<uint32_t>(rn) << 16 | - static_cast<uint32_t>(rt) << 12 | - static_cast<uint32_t>(rt2) << 8 | - B6 | B5 | B4 | - static_cast<uint32_t>(rd); - Emit32(encoding); -} - - -void Thumb2Assembler::clrex(Condition cond) { - CheckCondition(cond); - int32_t encoding = B31 | B30 | B29 | B28 | B25 | B24 | B23 | - B21 | B20 | - 0xf << 16 | - B15 | - 0xf << 8 | - B5 | - 0xf; - Emit32(encoding); -} - - -void Thumb2Assembler::nop(Condition cond) { - CheckCondition(cond); - uint16_t encoding = B15 | B13 | B12 | - B11 | B10 | B9 | B8; - Emit16(static_cast<int16_t>(encoding)); -} - - -void Thumb2Assembler::vmovsr(SRegister sn, Register rt, Condition cond) { - CHECK_NE(sn, kNoSRegister); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt, SP); - CHECK_NE(rt, PC); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B25 | - ((static_cast<int32_t>(sn) >> 1)*B16) | - (static_cast<int32_t>(rt)*B12) | B11 | B9 | - ((static_cast<int32_t>(sn) & 1)*B7) | B4; - Emit32(encoding); -} - - -void Thumb2Assembler::vmovrs(Register rt, SRegister sn, Condition cond) { - CHECK_NE(sn, kNoSRegister); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt, SP); - CHECK_NE(rt, PC); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B25 | B20 | - ((static_cast<int32_t>(sn) >> 1)*B16) | - (static_cast<int32_t>(rt)*B12) | B11 | B9 | - ((static_cast<int32_t>(sn) & 1)*B7) | B4; - Emit32(encoding); -} - - -void Thumb2Assembler::vmovsrr(SRegister sm, Register rt, Register rt2, - Condition cond) { - CHECK_NE(sm, kNoSRegister); - CHECK_NE(sm, S31); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt, SP); - CHECK_NE(rt, PC); - CHECK_NE(rt2, kNoRegister); - CHECK_NE(rt2, SP); - CHECK_NE(rt2, PC); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B22 | - (static_cast<int32_t>(rt2)*B16) | - (static_cast<int32_t>(rt)*B12) | B11 | B9 | - ((static_cast<int32_t>(sm) & 1)*B5) | B4 | - (static_cast<int32_t>(sm) >> 1); - Emit32(encoding); -} - - -void Thumb2Assembler::vmovrrs(Register rt, Register rt2, SRegister sm, - Condition cond) { - CHECK_NE(sm, kNoSRegister); - CHECK_NE(sm, S31); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt, SP); - CHECK_NE(rt, PC); - CHECK_NE(rt2, kNoRegister); - CHECK_NE(rt2, SP); - CHECK_NE(rt2, PC); - CHECK_NE(rt, rt2); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B22 | B20 | - (static_cast<int32_t>(rt2)*B16) | - (static_cast<int32_t>(rt)*B12) | B11 | B9 | - ((static_cast<int32_t>(sm) & 1)*B5) | B4 | - (static_cast<int32_t>(sm) >> 1); - Emit32(encoding); -} - - -void Thumb2Assembler::vmovdrr(DRegister dm, Register rt, Register rt2, - Condition cond) { - CHECK_NE(dm, kNoDRegister); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt, SP); - CHECK_NE(rt, PC); - CHECK_NE(rt2, kNoRegister); - CHECK_NE(rt2, SP); - CHECK_NE(rt2, PC); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B22 | - (static_cast<int32_t>(rt2)*B16) | - (static_cast<int32_t>(rt)*B12) | B11 | B9 | B8 | - ((static_cast<int32_t>(dm) >> 4)*B5) | B4 | - (static_cast<int32_t>(dm) & 0xf); - Emit32(encoding); -} - - -void Thumb2Assembler::vmovrrd(Register rt, Register rt2, DRegister dm, - Condition cond) { - CHECK_NE(dm, kNoDRegister); - CHECK_NE(rt, kNoRegister); - CHECK_NE(rt, SP); - CHECK_NE(rt, PC); - CHECK_NE(rt2, kNoRegister); - CHECK_NE(rt2, SP); - CHECK_NE(rt2, PC); - CHECK_NE(rt, rt2); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B22 | B20 | - (static_cast<int32_t>(rt2)*B16) | - (static_cast<int32_t>(rt)*B12) | B11 | B9 | B8 | - ((static_cast<int32_t>(dm) >> 4)*B5) | B4 | - (static_cast<int32_t>(dm) & 0xf); - Emit32(encoding); -} - - -void Thumb2Assembler::vldrs(SRegister sd, const Address& ad, Condition cond) { - const Address& addr = static_cast<const Address&>(ad); - CHECK_NE(sd, kNoSRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B24 | B20 | - ((static_cast<int32_t>(sd) & 1)*B22) | - ((static_cast<int32_t>(sd) >> 1)*B12) | - B11 | B9 | addr.vencoding(); - Emit32(encoding); -} - - -void Thumb2Assembler::vstrs(SRegister sd, const Address& ad, Condition cond) { - const Address& addr = static_cast<const Address&>(ad); - CHECK_NE(static_cast<Register>(addr.encodingArm() & (0xf << kRnShift)), PC); - CHECK_NE(sd, kNoSRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B24 | - ((static_cast<int32_t>(sd) & 1)*B22) | - ((static_cast<int32_t>(sd) >> 1)*B12) | - B11 | B9 | addr.vencoding(); - Emit32(encoding); -} - - -void Thumb2Assembler::vldrd(DRegister dd, const Address& ad, Condition cond) { - const Address& addr = static_cast<const Address&>(ad); - CHECK_NE(dd, kNoDRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B24 | B20 | - ((static_cast<int32_t>(dd) >> 4)*B22) | - ((static_cast<int32_t>(dd) & 0xf)*B12) | - B11 | B9 | B8 | addr.vencoding(); - Emit32(encoding); -} - - -void Thumb2Assembler::vstrd(DRegister dd, const Address& ad, Condition cond) { - const Address& addr = static_cast<const Address&>(ad); - CHECK_NE(static_cast<Register>(addr.encodingArm() & (0xf << kRnShift)), PC); - CHECK_NE(dd, kNoDRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B24 | - ((static_cast<int32_t>(dd) >> 4)*B22) | - ((static_cast<int32_t>(dd) & 0xf)*B12) | - B11 | B9 | B8 | addr.vencoding(); - Emit32(encoding); -} - - -void Thumb2Assembler::vpushs(SRegister reg, int nregs, Condition cond) { - EmitVPushPop(static_cast<uint32_t>(reg), nregs, true, false, cond); -} - - -void Thumb2Assembler::vpushd(DRegister reg, int nregs, Condition cond) { - EmitVPushPop(static_cast<uint32_t>(reg), nregs, true, true, cond); -} - - -void Thumb2Assembler::vpops(SRegister reg, int nregs, Condition cond) { - EmitVPushPop(static_cast<uint32_t>(reg), nregs, false, false, cond); -} - - -void Thumb2Assembler::vpopd(DRegister reg, int nregs, Condition cond) { - EmitVPushPop(static_cast<uint32_t>(reg), nregs, false, true, cond); -} - - -void Thumb2Assembler::vldmiad(Register base_reg, DRegister reg, int nregs, Condition cond) { - int32_t rest = B23; - EmitVLdmOrStm(rest, - static_cast<uint32_t>(reg), - nregs, - base_reg, - /*is_load*/ true, - /*dbl*/ true, - cond); -} - - -void Thumb2Assembler::vstmiad(Register base_reg, DRegister reg, int nregs, Condition cond) { - int32_t rest = B23; - EmitVLdmOrStm(rest, - static_cast<uint32_t>(reg), - nregs, - base_reg, - /*is_load*/ false, - /*dbl*/ true, - cond); -} - - -void Thumb2Assembler::EmitVPushPop(uint32_t reg, int nregs, bool push, bool dbl, Condition cond) { - int32_t rest = B21 | (push ? B24 : B23); - EmitVLdmOrStm(rest, reg, nregs, SP, /*is_load*/ !push, dbl, cond); -} - - -void Thumb2Assembler::EmitVLdmOrStm(int32_t rest, - uint32_t reg, - int nregs, - Register rn, - bool is_load, - bool dbl, - Condition cond) { - CheckCondition(cond); - - DCHECK_GT(nregs, 0); - DCHECK_LE(reg + nregs, 32u); - DCHECK(!dbl || (nregs <= 16)); - - uint32_t D; - uint32_t Vd; - if (dbl) { - // Encoded as D:Vd. - D = (reg >> 4) & 1; - Vd = reg & 15U /* 0b1111 */; - } else { - // Encoded as Vd:D. - D = reg & 1; - Vd = (reg >> 1) & 15U /* 0b1111 */; - } - - int32_t encoding = rest | - 14U /* 0b1110 */ << 28 | - B27 | B26 | B11 | B9 | - (is_load ? B20 : 0) | - static_cast<int16_t>(rn) << 16 | - D << 22 | - Vd << 12 | - (dbl ? B8 : 0) | - nregs << (dbl ? 1 : 0); - - Emit32(encoding); -} - - -void Thumb2Assembler::EmitVFPsss(Condition cond, int32_t opcode, - SRegister sd, SRegister sn, SRegister sm) { - CHECK_NE(sd, kNoSRegister); - CHECK_NE(sn, kNoSRegister); - CHECK_NE(sm, kNoSRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B25 | B11 | B9 | opcode | - ((static_cast<int32_t>(sd) & 1)*B22) | - ((static_cast<int32_t>(sn) >> 1)*B16) | - ((static_cast<int32_t>(sd) >> 1)*B12) | - ((static_cast<int32_t>(sn) & 1)*B7) | - ((static_cast<int32_t>(sm) & 1)*B5) | - (static_cast<int32_t>(sm) >> 1); - Emit32(encoding); -} - - -void Thumb2Assembler::EmitVFPddd(Condition cond, int32_t opcode, - DRegister dd, DRegister dn, DRegister dm) { - CHECK_NE(dd, kNoDRegister); - CHECK_NE(dn, kNoDRegister); - CHECK_NE(dm, kNoDRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B25 | B11 | B9 | B8 | opcode | - ((static_cast<int32_t>(dd) >> 4)*B22) | - ((static_cast<int32_t>(dn) & 0xf)*B16) | - ((static_cast<int32_t>(dd) & 0xf)*B12) | - ((static_cast<int32_t>(dn) >> 4)*B7) | - ((static_cast<int32_t>(dm) >> 4)*B5) | - (static_cast<int32_t>(dm) & 0xf); - Emit32(encoding); -} - - -void Thumb2Assembler::EmitVFPsd(Condition cond, int32_t opcode, - SRegister sd, DRegister dm) { - CHECK_NE(sd, kNoSRegister); - CHECK_NE(dm, kNoDRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B25 | B11 | B9 | opcode | - ((static_cast<int32_t>(sd) & 1)*B22) | - ((static_cast<int32_t>(sd) >> 1)*B12) | - ((static_cast<int32_t>(dm) >> 4)*B5) | - (static_cast<int32_t>(dm) & 0xf); - Emit32(encoding); -} - - -void Thumb2Assembler::EmitVFPds(Condition cond, int32_t opcode, - DRegister dd, SRegister sm) { - CHECK_NE(dd, kNoDRegister); - CHECK_NE(sm, kNoSRegister); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B25 | B11 | B9 | opcode | - ((static_cast<int32_t>(dd) >> 4)*B22) | - ((static_cast<int32_t>(dd) & 0xf)*B12) | - ((static_cast<int32_t>(sm) & 1)*B5) | - (static_cast<int32_t>(sm) >> 1); - Emit32(encoding); -} - - -void Thumb2Assembler::vmstat(Condition cond) { // VMRS APSR_nzcv, FPSCR. - CHECK_NE(cond, kNoCondition); - CheckCondition(cond); - int32_t encoding = (static_cast<int32_t>(cond) << kConditionShift) | - B27 | B26 | B25 | B23 | B22 | B21 | B20 | B16 | - (static_cast<int32_t>(PC)*B12) | - B11 | B9 | B4; - Emit32(encoding); -} - -void Thumb2Assembler::vcntd(DRegister dd, DRegister dm) { - uint32_t encoding = (B31 | B30 | B29 | B28 | B27 | B26 | B25 | B24 | B23 | B21 | B20) | - ((static_cast<int32_t>(dd) >> 4) * B22) | - ((static_cast<uint32_t>(dd) & 0xf) * B12) | - (B10 | B8) | - ((static_cast<int32_t>(dm) >> 4) * B5) | - (static_cast<uint32_t>(dm) & 0xf); - - Emit32(encoding); -} - -void Thumb2Assembler::vpaddld(DRegister dd, DRegister dm, int32_t size, bool is_unsigned) { - CHECK(size == 8 || size == 16 || size == 32) << size; - uint32_t encoding = (B31 | B30 | B29 | B28 | B27 | B26 | B25 | B24 | B23 | B21 | B20) | - ((static_cast<uint32_t>(size >> 4) & 0x3) * B18) | - ((static_cast<int32_t>(dd) >> 4) * B22) | - ((static_cast<uint32_t>(dd) & 0xf) * B12) | - (B9) | - (is_unsigned ? B7 : 0) | - ((static_cast<int32_t>(dm) >> 4) * B5) | - (static_cast<uint32_t>(dm) & 0xf); - - Emit32(encoding); -} - -void Thumb2Assembler::svc(uint32_t imm8) { - CHECK(IsUint<8>(imm8)) << imm8; - int16_t encoding = B15 | B14 | B12 | - B11 | B10 | B9 | B8 | - imm8; - Emit16(encoding); -} - - -void Thumb2Assembler::bkpt(uint16_t imm8) { - CHECK(IsUint<8>(imm8)) << imm8; - int16_t encoding = B15 | B13 | B12 | - B11 | B10 | B9 | - imm8; - Emit16(encoding); -} - -// Convert the given IT state to a mask bit given bit 0 of the first -// condition and a shift position. -static uint8_t ToItMask(ItState s, uint8_t firstcond0, uint8_t shift) { - switch (s) { - case kItOmitted: return 1 << shift; - case kItThen: return firstcond0 << shift; - case kItElse: return !firstcond0 << shift; - } - return 0; -} - - -// Set the IT condition in the given position for the given state. This is used -// to check that conditional instructions match the preceding IT statement. -void Thumb2Assembler::SetItCondition(ItState s, Condition cond, uint8_t index) { - switch (s) { - case kItOmitted: it_conditions_[index] = AL; break; - case kItThen: it_conditions_[index] = cond; break; - case kItElse: - it_conditions_[index] = static_cast<Condition>(static_cast<uint8_t>(cond) ^ 1); - break; - } -} - - -void Thumb2Assembler::it(Condition firstcond, ItState i1, ItState i2, ItState i3) { - CheckCondition(AL); // Not allowed in IT block. - uint8_t firstcond0 = static_cast<uint8_t>(firstcond) & 1; - - // All conditions to AL. - for (uint8_t i = 0; i < 4; ++i) { - it_conditions_[i] = AL; - } - - SetItCondition(kItThen, firstcond, 0); - uint8_t mask = ToItMask(i1, firstcond0, 3); - SetItCondition(i1, firstcond, 1); - - if (i1 != kItOmitted) { - mask |= ToItMask(i2, firstcond0, 2); - SetItCondition(i2, firstcond, 2); - if (i2 != kItOmitted) { - mask |= ToItMask(i3, firstcond0, 1); - SetItCondition(i3, firstcond, 3); - if (i3 != kItOmitted) { - mask |= 1U /* 0b0001 */; - } - } - } - - // Start at first condition. - it_cond_index_ = 0; - next_condition_ = it_conditions_[0]; - uint16_t encoding = B15 | B13 | B12 | - B11 | B10 | B9 | B8 | - firstcond << 4 | - mask; - Emit16(encoding); -} - - -void Thumb2Assembler::cbz(Register rn, Label* label) { - CheckCondition(AL); - if (label->IsBound()) { - LOG(FATAL) << "cbz can only be used to branch forwards"; - UNREACHABLE(); - } else if (IsHighRegister(rn)) { - LOG(FATAL) << "cbz can only be used with low registers"; - UNREACHABLE(); - } else { - uint16_t branchid = EmitCompareAndBranch(rn, static_cast<uint16_t>(label->position_), false); - label->LinkTo(branchid); - } -} - - -void Thumb2Assembler::cbnz(Register rn, Label* label) { - CheckCondition(AL); - if (label->IsBound()) { - LOG(FATAL) << "cbnz can only be used to branch forwards"; - UNREACHABLE(); - } else if (IsHighRegister(rn)) { - LOG(FATAL) << "cbnz can only be used with low registers"; - UNREACHABLE(); - } else { - uint16_t branchid = EmitCompareAndBranch(rn, static_cast<uint16_t>(label->position_), true); - label->LinkTo(branchid); - } -} - - -void Thumb2Assembler::blx(Register rm, Condition cond) { - CHECK_NE(rm, kNoRegister); - CheckCondition(cond); - int16_t encoding = B14 | B10 | B9 | B8 | B7 | static_cast<int16_t>(rm) << 3; - Emit16(encoding); -} - - -void Thumb2Assembler::bx(Register rm, Condition cond) { - CHECK_NE(rm, kNoRegister); - CheckCondition(cond); - int16_t encoding = B14 | B10 | B9 | B8 | static_cast<int16_t>(rm) << 3; - Emit16(encoding); -} - - -void Thumb2Assembler::AdrCode(Register rt, Label* label) { - uint32_t pc = buffer_.Size(); - FixupId branch_id = AddFixup(Fixup::LoadCodeAddress(pc, rt)); - CHECK(!label->IsBound()); - // ADR target must be an unbound label. Add it to a singly-linked list maintained within - // the code with the label serving as the head. - Emit16(static_cast<uint16_t>(label->position_)); - label->LinkTo(branch_id); - Emit16(0); - DCHECK_EQ(buffer_.Size() - pc, GetFixup(branch_id)->GetSizeInBytes()); -} - - -void Thumb2Assembler::Push(Register rd, Condition cond) { - str(rd, Address(SP, -kRegisterSize, Address::PreIndex), cond); -} - - -void Thumb2Assembler::Pop(Register rd, Condition cond) { - ldr(rd, Address(SP, kRegisterSize, Address::PostIndex), cond); -} - - -void Thumb2Assembler::PushList(RegList regs, Condition cond) { - stm(DB_W, SP, regs, cond); -} - - -void Thumb2Assembler::PopList(RegList regs, Condition cond) { - ldm(IA_W, SP, regs, cond); -} - -void Thumb2Assembler::StoreList(RegList regs, size_t stack_offset) { - DCHECK_NE(regs, 0u); - DCHECK_EQ(regs & (1u << IP), 0u); - if (IsPowerOfTwo(regs)) { - Register reg = static_cast<Register>(CTZ(static_cast<uint32_t>(regs))); - str(reg, Address(SP, stack_offset)); - } else { - add(IP, SP, ShifterOperand(stack_offset)); - stm(IA, IP, regs); - } -} - -void Thumb2Assembler::LoadList(RegList regs, size_t stack_offset) { - DCHECK_NE(regs, 0u); - DCHECK_EQ(regs & (1u << IP), 0u); - if (IsPowerOfTwo(regs)) { - Register reg = static_cast<Register>(CTZ(static_cast<uint32_t>(regs))); - ldr(reg, Address(SP, stack_offset)); - } else { - Register lowest_reg = static_cast<Register>(CTZ(static_cast<uint32_t>(regs))); - add(lowest_reg, SP, ShifterOperand(stack_offset)); - ldm(IA, lowest_reg, regs); - } -} - -void Thumb2Assembler::Mov(Register rd, Register rm, Condition cond) { - if (cond != AL || rd != rm) { - mov(rd, ShifterOperand(rm), cond); - } -} - - -void Thumb2Assembler::Bind(Label* label) { - BindLabel(label, buffer_.Size()); - - // Try to emit some Fixups now to reduce the memory needed during the branch fixup later. - while (!fixups_.empty() && fixups_.back().IsCandidateForEmitEarly()) { - const Fixup& last_fixup = fixups_.back(); - // Fixups are ordered by location, so the candidate can surely be emitted if it is - // a forward branch. If it's a backward branch, it may go over any number of other - // fixups. We could check for any number of emit early candidates but we want this - // heuristics to be quick, so check just one. - uint32_t target = last_fixup.GetTarget(); - if (target < last_fixup.GetLocation() && - fixups_.size() >= 2u && - fixups_[fixups_.size() - 2u].GetLocation() >= target) { - const Fixup& prev_fixup = fixups_[fixups_.size() - 2u]; - if (!prev_fixup.IsCandidateForEmitEarly()) { - break; - } - uint32_t min_target = std::min(target, prev_fixup.GetTarget()); - if (fixups_.size() >= 3u && fixups_[fixups_.size() - 3u].GetLocation() >= min_target) { - break; - } - } - last_fixup.Emit(last_fixup.GetLocation(), &buffer_, buffer_.Size()); - fixups_.pop_back(); - } -} - - -void Thumb2Assembler::Lsl(Register rd, Register rm, uint32_t shift_imm, - Condition cond, SetCc set_cc) { - CHECK_LE(shift_imm, 31u); - CheckCondition(cond); - EmitShift(rd, rm, LSL, shift_imm, cond, set_cc); -} - - -void Thumb2Assembler::Lsr(Register rd, Register rm, uint32_t shift_imm, - Condition cond, SetCc set_cc) { - CHECK(1u <= shift_imm && shift_imm <= 32u); - if (shift_imm == 32) shift_imm = 0; // Comply to UAL syntax. - CheckCondition(cond); - EmitShift(rd, rm, LSR, shift_imm, cond, set_cc); -} - - -void Thumb2Assembler::Asr(Register rd, Register rm, uint32_t shift_imm, - Condition cond, SetCc set_cc) { - CHECK(1u <= shift_imm && shift_imm <= 32u); - if (shift_imm == 32) shift_imm = 0; // Comply to UAL syntax. - CheckCondition(cond); - EmitShift(rd, rm, ASR, shift_imm, cond, set_cc); -} - - -void Thumb2Assembler::Ror(Register rd, Register rm, uint32_t shift_imm, - Condition cond, SetCc set_cc) { - CHECK(1u <= shift_imm && shift_imm <= 31u); - CheckCondition(cond); - EmitShift(rd, rm, ROR, shift_imm, cond, set_cc); -} - - -void Thumb2Assembler::Rrx(Register rd, Register rm, Condition cond, SetCc set_cc) { - CheckCondition(cond); - EmitShift(rd, rm, RRX, 0, cond, set_cc); -} - - -void Thumb2Assembler::Lsl(Register rd, Register rm, Register rn, - Condition cond, SetCc set_cc) { - CheckCondition(cond); - EmitShift(rd, rm, LSL, rn, cond, set_cc); -} - - -void Thumb2Assembler::Lsr(Register rd, Register rm, Register rn, - Condition cond, SetCc set_cc) { - CheckCondition(cond); - EmitShift(rd, rm, LSR, rn, cond, set_cc); -} - - -void Thumb2Assembler::Asr(Register rd, Register rm, Register rn, - Condition cond, SetCc set_cc) { - CheckCondition(cond); - EmitShift(rd, rm, ASR, rn, cond, set_cc); -} - - -void Thumb2Assembler::Ror(Register rd, Register rm, Register rn, - Condition cond, SetCc set_cc) { - CheckCondition(cond); - EmitShift(rd, rm, ROR, rn, cond, set_cc); -} - - -int32_t Thumb2Assembler::EncodeBranchOffset(int32_t offset, int32_t inst) { - // The offset is off by 4 due to the way the ARM CPUs read PC. - offset -= 4; - offset >>= 1; - - uint32_t value = 0; - // There are two different encodings depending on the value of bit 12. In one case - // intermediate values are calculated using the sign bit. - if ((inst & B12) == B12) { - // 25 bits of offset. - uint32_t signbit = (offset >> 31) & 0x1; - uint32_t i1 = (offset >> 22) & 0x1; - uint32_t i2 = (offset >> 21) & 0x1; - uint32_t imm10 = (offset >> 11) & 0x03ff; - uint32_t imm11 = offset & 0x07ff; - uint32_t j1 = (i1 ^ signbit) ? 0 : 1; - uint32_t j2 = (i2 ^ signbit) ? 0 : 1; - value = (signbit << 26) | (j1 << 13) | (j2 << 11) | (imm10 << 16) | - imm11; - // Remove the offset from the current encoding. - inst &= ~(0x3ff << 16 | 0x7ff); - } else { - uint32_t signbit = (offset >> 31) & 0x1; - uint32_t imm6 = (offset >> 11) & 0x03f; - uint32_t imm11 = offset & 0x07ff; - uint32_t j1 = (offset >> 19) & 1; - uint32_t j2 = (offset >> 17) & 1; - value = (signbit << 26) | (j1 << 13) | (j2 << 11) | (imm6 << 16) | - imm11; - // Remove the offset from the current encoding. - inst &= ~(0x3f << 16 | 0x7ff); - } - // Mask out offset bits in current instruction. - inst &= ~(B26 | B13 | B11); - inst |= value; - return inst; -} - - -int Thumb2Assembler::DecodeBranchOffset(int32_t instr) { - int32_t imm32; - if ((instr & B12) == B12) { - uint32_t S = (instr >> 26) & 1; - uint32_t J2 = (instr >> 11) & 1; - uint32_t J1 = (instr >> 13) & 1; - uint32_t imm10 = (instr >> 16) & 0x3FF; - uint32_t imm11 = instr & 0x7FF; - - uint32_t I1 = ~(J1 ^ S) & 1; - uint32_t I2 = ~(J2 ^ S) & 1; - imm32 = (S << 24) | (I1 << 23) | (I2 << 22) | (imm10 << 12) | (imm11 << 1); - imm32 = (imm32 << 8) >> 8; // sign extend 24 bit immediate. - } else { - uint32_t S = (instr >> 26) & 1; - uint32_t J2 = (instr >> 11) & 1; - uint32_t J1 = (instr >> 13) & 1; - uint32_t imm6 = (instr >> 16) & 0x3F; - uint32_t imm11 = instr & 0x7FF; - - imm32 = (S << 20) | (J2 << 19) | (J1 << 18) | (imm6 << 12) | (imm11 << 1); - imm32 = (imm32 << 11) >> 11; // sign extend 21 bit immediate. - } - imm32 += 4; - return imm32; -} - -uint32_t Thumb2Assembler::GetAdjustedPosition(uint32_t old_position) { - // We can reconstruct the adjustment by going through all the fixups from the beginning - // up to the old_position. Since we expect AdjustedPosition() to be called in a loop - // with increasing old_position, we can use the data from last AdjustedPosition() to - // continue where we left off and the whole loop should be O(m+n) where m is the number - // of positions to adjust and n is the number of fixups. - if (old_position < last_old_position_) { - last_position_adjustment_ = 0u; - last_old_position_ = 0u; - last_fixup_id_ = 0u; - } - while (last_fixup_id_ != fixups_.size()) { - Fixup* fixup = GetFixup(last_fixup_id_); - if (fixup->GetLocation() >= old_position + last_position_adjustment_) { - break; - } - if (fixup->GetSize() != fixup->GetOriginalSize()) { - last_position_adjustment_ += fixup->GetSizeInBytes() - fixup->GetOriginalSizeInBytes(); - } - ++last_fixup_id_; - } - last_old_position_ = old_position; - return old_position + last_position_adjustment_; -} - -Literal* Thumb2Assembler::NewLiteral(size_t size, const uint8_t* data) { - DCHECK(size == 4u || size == 8u) << size; - literals_.emplace_back(size, data); - return &literals_.back(); -} - -void Thumb2Assembler::LoadLiteral(Register rt, Literal* literal) { - DCHECK_EQ(literal->GetSize(), 4u); - DCHECK(!literal->GetLabel()->IsBound()); - bool use32bit = IsForced32Bit() || IsHighRegister(rt); - uint32_t location = buffer_.Size(); - Fixup::Size size = use32bit ? Fixup::kLiteral4KiB : Fixup::kLiteral1KiB; - FixupId fixup_id = AddFixup(Fixup::LoadNarrowLiteral(location, rt, size)); - Emit16(static_cast<uint16_t>(literal->GetLabel()->position_)); - literal->GetLabel()->LinkTo(fixup_id); - if (use32bit) { - Emit16(0); - } - DCHECK_EQ(location + GetFixup(fixup_id)->GetSizeInBytes(), buffer_.Size()); -} - -void Thumb2Assembler::LoadLiteral(Register rt, Register rt2, Literal* literal) { - DCHECK_EQ(literal->GetSize(), 8u); - DCHECK(!literal->GetLabel()->IsBound()); - uint32_t location = buffer_.Size(); - FixupId fixup_id = - AddFixup(Fixup::LoadWideLiteral(location, rt, rt2, Fixup::kLongOrFPLiteral1KiB)); - Emit16(static_cast<uint16_t>(literal->GetLabel()->position_)); - literal->GetLabel()->LinkTo(fixup_id); - Emit16(0); - DCHECK_EQ(location + GetFixup(fixup_id)->GetSizeInBytes(), buffer_.Size()); -} - -void Thumb2Assembler::LoadLiteral(SRegister sd, Literal* literal) { - DCHECK_EQ(literal->GetSize(), 4u); - DCHECK(!literal->GetLabel()->IsBound()); - uint32_t location = buffer_.Size(); - FixupId fixup_id = AddFixup(Fixup::LoadSingleLiteral(location, sd, Fixup::kLongOrFPLiteral1KiB)); - Emit16(static_cast<uint16_t>(literal->GetLabel()->position_)); - literal->GetLabel()->LinkTo(fixup_id); - Emit16(0); - DCHECK_EQ(location + GetFixup(fixup_id)->GetSizeInBytes(), buffer_.Size()); -} - -void Thumb2Assembler::LoadLiteral(DRegister dd, Literal* literal) { - DCHECK_EQ(literal->GetSize(), 8u); - DCHECK(!literal->GetLabel()->IsBound()); - uint32_t location = buffer_.Size(); - FixupId fixup_id = AddFixup(Fixup::LoadDoubleLiteral(location, dd, Fixup::kLongOrFPLiteral1KiB)); - Emit16(static_cast<uint16_t>(literal->GetLabel()->position_)); - literal->GetLabel()->LinkTo(fixup_id); - Emit16(0); - DCHECK_EQ(location + GetFixup(fixup_id)->GetSizeInBytes(), buffer_.Size()); -} - - -void Thumb2Assembler::AddConstant(Register rd, Register rn, int32_t value, - Condition cond, SetCc set_cc) { - if (value == 0 && set_cc != kCcSet) { - if (rd != rn) { - mov(rd, ShifterOperand(rn), cond); - } - return; - } - // We prefer to select the shorter code sequence rather than selecting add for - // positive values and sub for negatives ones, which would slightly improve - // the readability of generated code for some constants. - ShifterOperand shifter_op; - if (ShifterOperandCanHold(rd, rn, ADD, value, set_cc, &shifter_op)) { - add(rd, rn, shifter_op, cond, set_cc); - } else if (ShifterOperandCanHold(rd, rn, SUB, -value, set_cc, &shifter_op)) { - sub(rd, rn, shifter_op, cond, set_cc); - } else { - CHECK(rn != IP); - // If rd != rn, use rd as temp. This alows 16-bit ADD/SUB in more situations than using IP. - Register temp = (rd != rn) ? rd : IP; - if (ShifterOperandCanHold(temp, kNoRegister, MVN, ~value, kCcKeep, &shifter_op)) { - mvn(temp, shifter_op, cond, kCcKeep); - add(rd, rn, ShifterOperand(temp), cond, set_cc); - } else if (ShifterOperandCanHold(temp, kNoRegister, MVN, ~(-value), kCcKeep, &shifter_op)) { - mvn(temp, shifter_op, cond, kCcKeep); - sub(rd, rn, ShifterOperand(temp), cond, set_cc); - } else if (High16Bits(-value) == 0) { - movw(temp, Low16Bits(-value), cond); - sub(rd, rn, ShifterOperand(temp), cond, set_cc); - } else { - movw(temp, Low16Bits(value), cond); - uint16_t value_high = High16Bits(value); - if (value_high != 0) { - movt(temp, value_high, cond); - } - add(rd, rn, ShifterOperand(temp), cond, set_cc); - } - } -} - -void Thumb2Assembler::CmpConstant(Register rn, int32_t value, Condition cond) { - // We prefer to select the shorter code sequence rather than using plain cmp and cmn - // which would slightly improve the readability of generated code for some constants. - ShifterOperand shifter_op; - if (ShifterOperandCanHold(kNoRegister, rn, CMP, value, kCcSet, &shifter_op)) { - cmp(rn, shifter_op, cond); - } else if (ShifterOperandCanHold(kNoRegister, rn, CMN, -value, kCcSet, &shifter_op)) { - cmn(rn, shifter_op, cond); - } else { - CHECK(rn != IP); - if (ShifterOperandCanHold(IP, kNoRegister, MVN, ~value, kCcKeep, &shifter_op)) { - mvn(IP, shifter_op, cond, kCcKeep); - cmp(rn, ShifterOperand(IP), cond); - } else if (ShifterOperandCanHold(IP, kNoRegister, MVN, ~(-value), kCcKeep, &shifter_op)) { - mvn(IP, shifter_op, cond, kCcKeep); - cmn(rn, ShifterOperand(IP), cond); - } else if (High16Bits(-value) == 0) { - movw(IP, Low16Bits(-value), cond); - cmn(rn, ShifterOperand(IP), cond); - } else { - movw(IP, Low16Bits(value), cond); - uint16_t value_high = High16Bits(value); - if (value_high != 0) { - movt(IP, value_high, cond); - } - cmp(rn, ShifterOperand(IP), cond); - } - } -} - -void Thumb2Assembler::LoadImmediate(Register rd, int32_t value, Condition cond) { - ShifterOperand shifter_op; - if (ShifterOperandCanHold(rd, R0, MOV, value, &shifter_op)) { - mov(rd, shifter_op, cond); - } else if (ShifterOperandCanHold(rd, R0, MVN, ~value, &shifter_op)) { - mvn(rd, shifter_op, cond); - } else { - movw(rd, Low16Bits(value), cond); - uint16_t value_high = High16Bits(value); - if (value_high != 0) { - movt(rd, value_high, cond); - } - } -} - -void Thumb2Assembler::LoadDImmediate(DRegister dd, double value, Condition cond) { - if (!vmovd(dd, value, cond)) { - uint64_t int_value = bit_cast<uint64_t, double>(value); - if (int_value == bit_cast<uint64_t, double>(0.0)) { - // 0.0 is quite common, so we special case it by loading - // 2.0 in `dd` and then subtracting it. - bool success = vmovd(dd, 2.0, cond); - CHECK(success); - vsubd(dd, dd, dd, cond); - } else { - Literal* literal = literal64_dedupe_map_.GetOrCreate( - int_value, - [this, int_value]() { return NewLiteral<uint64_t>(int_value); }); - LoadLiteral(dd, literal); - } - } -} - -int32_t Thumb2Assembler::GetAllowedLoadOffsetBits(LoadOperandType type) { - switch (type) { - case kLoadSignedByte: - case kLoadSignedHalfword: - case kLoadUnsignedHalfword: - case kLoadUnsignedByte: - case kLoadWord: - // We can encode imm12 offset. - return 0xfffu; - case kLoadSWord: - case kLoadDWord: - case kLoadWordPair: - // We can encode imm8:'00' offset. - return 0xff << 2; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - -int32_t Thumb2Assembler::GetAllowedStoreOffsetBits(StoreOperandType type) { - switch (type) { - case kStoreHalfword: - case kStoreByte: - case kStoreWord: - // We can encode imm12 offset. - return 0xfff; - case kStoreSWord: - case kStoreDWord: - case kStoreWordPair: - // We can encode imm8:'00' offset. - return 0xff << 2; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - -bool Thumb2Assembler::CanSplitLoadStoreOffset(int32_t allowed_offset_bits, - int32_t offset, - /*out*/ int32_t* add_to_base, - /*out*/ int32_t* offset_for_load_store) { - int32_t other_bits = offset & ~allowed_offset_bits; - if (ShifterOperandCanAlwaysHold(other_bits) || ShifterOperandCanAlwaysHold(-other_bits)) { - *add_to_base = offset & ~allowed_offset_bits; - *offset_for_load_store = offset & allowed_offset_bits; - return true; - } - return false; -} - -int32_t Thumb2Assembler::AdjustLoadStoreOffset(int32_t allowed_offset_bits, - Register temp, - Register base, - int32_t offset, - Condition cond) { - DCHECK_NE(offset & ~allowed_offset_bits, 0); - int32_t add_to_base, offset_for_load; - if (CanSplitLoadStoreOffset(allowed_offset_bits, offset, &add_to_base, &offset_for_load)) { - AddConstant(temp, base, add_to_base, cond, kCcKeep); - return offset_for_load; - } else { - LoadImmediate(temp, offset, cond); - add(temp, temp, ShifterOperand(base), cond, kCcKeep); - return 0; - } -} - -// Implementation note: this method must emit at most one instruction when -// Address::CanHoldLoadOffsetThumb. -void Thumb2Assembler::LoadFromOffset(LoadOperandType type, - Register reg, - Register base, - int32_t offset, - Condition cond) { - if (!Address::CanHoldLoadOffsetThumb(type, offset)) { - CHECK_NE(base, IP); - // Inlined AdjustLoadStoreOffset() allows us to pull a few more tricks. - int32_t allowed_offset_bits = GetAllowedLoadOffsetBits(type); - DCHECK_NE(offset & ~allowed_offset_bits, 0); - int32_t add_to_base, offset_for_load; - if (CanSplitLoadStoreOffset(allowed_offset_bits, offset, &add_to_base, &offset_for_load)) { - // Use reg for the adjusted base. If it's low reg, we may end up using 16-bit load. - AddConstant(reg, base, add_to_base, cond, kCcKeep); - base = reg; - offset = offset_for_load; - } else { - Register temp = (reg == base) ? IP : reg; - LoadImmediate(temp, offset, cond); - // TODO: Implement indexed load (not available for LDRD) and use it here to avoid the ADD. - // Use reg for the adjusted base. If it's low reg, we may end up using 16-bit load. - add(reg, reg, ShifterOperand((reg == base) ? IP : base), cond, kCcKeep); - base = reg; - offset = 0; - } - } - DCHECK(Address::CanHoldLoadOffsetThumb(type, offset)); - switch (type) { - case kLoadSignedByte: - ldrsb(reg, Address(base, offset), cond); - break; - case kLoadUnsignedByte: - ldrb(reg, Address(base, offset), cond); - break; - case kLoadSignedHalfword: - ldrsh(reg, Address(base, offset), cond); - break; - case kLoadUnsignedHalfword: - ldrh(reg, Address(base, offset), cond); - break; - case kLoadWord: - ldr(reg, Address(base, offset), cond); - break; - case kLoadWordPair: - ldrd(reg, Address(base, offset), cond); - break; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - -// Implementation note: this method must emit at most one instruction when -// Address::CanHoldLoadOffsetThumb, as expected by JIT::GuardedLoadFromOffset. -void Thumb2Assembler::LoadSFromOffset(SRegister reg, - Register base, - int32_t offset, - Condition cond) { - if (!Address::CanHoldLoadOffsetThumb(kLoadSWord, offset)) { - CHECK_NE(base, IP); - offset = AdjustLoadStoreOffset(GetAllowedLoadOffsetBits(kLoadSWord), IP, base, offset, cond); - base = IP; - } - DCHECK(Address::CanHoldLoadOffsetThumb(kLoadSWord, offset)); - vldrs(reg, Address(base, offset), cond); -} - - -// Implementation note: this method must emit at most one instruction when -// Address::CanHoldLoadOffsetThumb, as expected by JIT::GuardedLoadFromOffset. -void Thumb2Assembler::LoadDFromOffset(DRegister reg, - Register base, - int32_t offset, - Condition cond) { - if (!Address::CanHoldLoadOffsetThumb(kLoadDWord, offset)) { - CHECK_NE(base, IP); - offset = AdjustLoadStoreOffset(GetAllowedLoadOffsetBits(kLoadDWord), IP, base, offset, cond); - base = IP; - } - DCHECK(Address::CanHoldLoadOffsetThumb(kLoadDWord, offset)); - vldrd(reg, Address(base, offset), cond); -} - - -// Implementation note: this method must emit at most one instruction when -// Address::CanHoldStoreOffsetThumb. -void Thumb2Assembler::StoreToOffset(StoreOperandType type, - Register reg, - Register base, - int32_t offset, - Condition cond) { - Register tmp_reg = kNoRegister; - if (!Address::CanHoldStoreOffsetThumb(type, offset)) { - CHECK_NE(base, IP); - if ((reg != IP) && - ((type != kStoreWordPair) || (reg + 1 != IP))) { - tmp_reg = IP; - } else { - // Be careful not to use IP twice (for `reg` (or `reg` + 1 in - // the case of a word-pair store) and `base`) to build the - // Address object used by the store instruction(s) below. - // Instead, save R5 on the stack (or R6 if R5 is already used by - // `base`), use it as secondary temporary register, and restore - // it after the store instruction has been emitted. - tmp_reg = (base != R5) ? R5 : R6; - Push(tmp_reg); - if (base == SP) { - offset += kRegisterSize; - } - } - // TODO: Implement indexed store (not available for STRD), inline AdjustLoadStoreOffset() - // and in the "unsplittable" path get rid of the "add" by using the store indexed instead. - offset = AdjustLoadStoreOffset(GetAllowedStoreOffsetBits(type), tmp_reg, base, offset, cond); - base = tmp_reg; - } - DCHECK(Address::CanHoldStoreOffsetThumb(type, offset)); - switch (type) { - case kStoreByte: - strb(reg, Address(base, offset), cond); - break; - case kStoreHalfword: - strh(reg, Address(base, offset), cond); - break; - case kStoreWord: - str(reg, Address(base, offset), cond); - break; - case kStoreWordPair: - strd(reg, Address(base, offset), cond); - break; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } - if ((tmp_reg != kNoRegister) && (tmp_reg != IP)) { - CHECK((tmp_reg == R5) || (tmp_reg == R6)); - Pop(tmp_reg); - } -} - - -// Implementation note: this method must emit at most one instruction when -// Address::CanHoldStoreOffsetThumb, as expected by JIT::GuardedStoreToOffset. -void Thumb2Assembler::StoreSToOffset(SRegister reg, - Register base, - int32_t offset, - Condition cond) { - if (!Address::CanHoldStoreOffsetThumb(kStoreSWord, offset)) { - CHECK_NE(base, IP); - offset = AdjustLoadStoreOffset(GetAllowedStoreOffsetBits(kStoreSWord), IP, base, offset, cond); - base = IP; - } - DCHECK(Address::CanHoldStoreOffsetThumb(kStoreSWord, offset)); - vstrs(reg, Address(base, offset), cond); -} - - -// Implementation note: this method must emit at most one instruction when -// Address::CanHoldStoreOffsetThumb, as expected by JIT::GuardedStoreSToOffset. -void Thumb2Assembler::StoreDToOffset(DRegister reg, - Register base, - int32_t offset, - Condition cond) { - if (!Address::CanHoldStoreOffsetThumb(kStoreDWord, offset)) { - CHECK_NE(base, IP); - offset = AdjustLoadStoreOffset(GetAllowedStoreOffsetBits(kStoreDWord), IP, base, offset, cond); - base = IP; - } - DCHECK(Address::CanHoldStoreOffsetThumb(kStoreDWord, offset)); - vstrd(reg, Address(base, offset), cond); -} - - -void Thumb2Assembler::dmb(DmbOptions flavor) { - int32_t encoding = 0xf3bf8f50; // dmb in T1 encoding. - Emit32(encoding | flavor); -} - - -void Thumb2Assembler::CompareAndBranchIfZero(Register r, Label* label) { - if (CanRelocateBranches() && IsLowRegister(r) && !label->IsBound()) { - cbz(r, label); - } else { - cmp(r, ShifterOperand(0)); - b(label, EQ); - } -} - - -void Thumb2Assembler::CompareAndBranchIfNonZero(Register r, Label* label) { - if (CanRelocateBranches() && IsLowRegister(r) && !label->IsBound()) { - cbnz(r, label); - } else { - cmp(r, ShifterOperand(0)); - b(label, NE); - } -} - -JumpTable* Thumb2Assembler::CreateJumpTable(std::vector<Label*>&& labels, Register base_reg) { - jump_tables_.emplace_back(std::move(labels)); - JumpTable* table = &jump_tables_.back(); - DCHECK(!table->GetLabel()->IsBound()); - - bool use32bit = IsForced32Bit() || IsHighRegister(base_reg); - uint32_t location = buffer_.Size(); - Fixup::Size size = use32bit ? Fixup::kLiteralAddr4KiB : Fixup::kLiteralAddr1KiB; - FixupId fixup_id = AddFixup(Fixup::LoadLiteralAddress(location, base_reg, size)); - Emit16(static_cast<uint16_t>(table->GetLabel()->position_)); - table->GetLabel()->LinkTo(fixup_id); - if (use32bit) { - Emit16(0); - } - DCHECK_EQ(location + GetFixup(fixup_id)->GetSizeInBytes(), buffer_.Size()); - - return table; -} - -void Thumb2Assembler::EmitJumpTableDispatch(JumpTable* jump_table, Register displacement_reg) { - CHECK(!IsForced32Bit()) << "Forced 32-bit dispatch not implemented yet"; - // 32-bit ADD doesn't support PC as an input, so we need a two-instruction sequence: - // SUB ip, ip, #0 - // ADD pc, ip, reg - // TODO: Implement. - - // The anchor's position needs to be fixed up before we can compute offsets - so make it a tracked - // label. - BindTrackedLabel(jump_table->GetAnchorLabel()); - - add(PC, PC, ShifterOperand(displacement_reg)); -} - -} // namespace arm -} // namespace art diff --git a/compiler/utils/arm/assembler_thumb2.h b/compiler/utils/arm/assembler_thumb2.h deleted file mode 100644 index 2ff9018510..0000000000 --- a/compiler/utils/arm/assembler_thumb2.h +++ /dev/null @@ -1,948 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_UTILS_ARM_ASSEMBLER_THUMB2_H_ -#define ART_COMPILER_UTILS_ARM_ASSEMBLER_THUMB2_H_ - -#include <deque> -#include <utility> -#include <vector> - -#include "base/arena_containers.h" -#include "base/array_ref.h" -#include "base/logging.h" -#include "constants_arm.h" -#include "utils/arm/managed_register_arm.h" -#include "utils/arm/assembler_arm.h" -#include "offsets.h" - -namespace art { -namespace arm { - -class Thumb2Assembler FINAL : public ArmAssembler { - public: - explicit Thumb2Assembler(ArenaAllocator* arena, bool can_relocate_branches = true) - : ArmAssembler(arena), - can_relocate_branches_(can_relocate_branches), - force_32bit_(false), - it_cond_index_(kNoItCondition), - next_condition_(AL), - fixups_(arena->Adapter(kArenaAllocAssembler)), - fixup_dependents_(arena->Adapter(kArenaAllocAssembler)), - literals_(arena->Adapter(kArenaAllocAssembler)), - literal64_dedupe_map_(std::less<uint64_t>(), arena->Adapter(kArenaAllocAssembler)), - jump_tables_(arena->Adapter(kArenaAllocAssembler)), - last_position_adjustment_(0u), - last_old_position_(0u), - last_fixup_id_(0u) { - cfi().DelayEmittingAdvancePCs(); - } - - virtual ~Thumb2Assembler() { - } - - bool IsThumb() const OVERRIDE { - return true; - } - - bool IsForced32Bit() const { - return force_32bit_; - } - - bool CanRelocateBranches() const { - return can_relocate_branches_; - } - - void FinalizeCode() OVERRIDE; - - // Data-processing instructions. - virtual void and_(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void eor(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void sub(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void rsb(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void add(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void adc(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void sbc(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void rsc(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - void tst(Register rn, const ShifterOperand& so, Condition cond = AL) OVERRIDE; - - void teq(Register rn, const ShifterOperand& so, Condition cond = AL) OVERRIDE; - - void cmp(Register rn, const ShifterOperand& so, Condition cond = AL) OVERRIDE; - - void cmn(Register rn, const ShifterOperand& so, Condition cond = AL) OVERRIDE; - - virtual void orr(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void orn(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void mov(Register rd, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void bic(Register rd, Register rn, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void mvn(Register rd, const ShifterOperand& so, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - // Miscellaneous data-processing instructions. - void clz(Register rd, Register rm, Condition cond = AL) OVERRIDE; - void movw(Register rd, uint16_t imm16, Condition cond = AL) OVERRIDE; - void movt(Register rd, uint16_t imm16, Condition cond = AL) OVERRIDE; - void rbit(Register rd, Register rm, Condition cond = AL) OVERRIDE; - void rev(Register rd, Register rm, Condition cond = AL) OVERRIDE; - void rev16(Register rd, Register rm, Condition cond = AL) OVERRIDE; - void revsh(Register rd, Register rm, Condition cond = AL) OVERRIDE; - - // Multiply instructions. - void mul(Register rd, Register rn, Register rm, Condition cond = AL) OVERRIDE; - void mla(Register rd, Register rn, Register rm, Register ra, - Condition cond = AL) OVERRIDE; - void mls(Register rd, Register rn, Register rm, Register ra, - Condition cond = AL) OVERRIDE; - void smull(Register rd_lo, Register rd_hi, Register rn, Register rm, - Condition cond = AL) OVERRIDE; - void umull(Register rd_lo, Register rd_hi, Register rn, Register rm, - Condition cond = AL) OVERRIDE; - - void sdiv(Register rd, Register rn, Register rm, Condition cond = AL) OVERRIDE; - void udiv(Register rd, Register rn, Register rm, Condition cond = AL) OVERRIDE; - - // Bit field extract instructions. - void sbfx(Register rd, Register rn, uint32_t lsb, uint32_t width, Condition cond = AL) OVERRIDE; - void ubfx(Register rd, Register rn, uint32_t lsb, uint32_t width, Condition cond = AL) OVERRIDE; - - // Load/store instructions. - void ldr(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - void str(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - - void ldrb(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - void strb(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - - void ldrh(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - void strh(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - - void ldrsb(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - void ldrsh(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - - // Load/store register dual instructions using registers `rd` and `rd` + 1. - void ldrd(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - void strd(Register rd, const Address& ad, Condition cond = AL) OVERRIDE; - - // Load/store register dual instructions using registers `rd` and `rd2`. - // Note that contrary to the ARM A1 encoding, the Thumb-2 T1 encoding - // does not require `rd` to be even, nor `rd2' to be equal to `rd` + 1. - void ldrd(Register rd, Register rd2, const Address& ad, Condition cond); - void strd(Register rd, Register rd2, const Address& ad, Condition cond); - - - void ldm(BlockAddressMode am, Register base, - RegList regs, Condition cond = AL) OVERRIDE; - void stm(BlockAddressMode am, Register base, - RegList regs, Condition cond = AL) OVERRIDE; - - void ldrex(Register rd, Register rn, Condition cond = AL) OVERRIDE; - void strex(Register rd, Register rt, Register rn, Condition cond = AL) OVERRIDE; - - void ldrex(Register rd, Register rn, uint16_t imm, Condition cond = AL); - void strex(Register rd, Register rt, Register rn, uint16_t imm, Condition cond = AL); - - void ldrexd(Register rt, Register rt2, Register rn, Condition cond = AL) OVERRIDE; - void strexd(Register rd, Register rt, Register rt2, Register rn, Condition cond = AL) OVERRIDE; - - // Miscellaneous instructions. - void clrex(Condition cond = AL) OVERRIDE; - void nop(Condition cond = AL) OVERRIDE; - - void bkpt(uint16_t imm16) OVERRIDE; - void svc(uint32_t imm24) OVERRIDE; - - // If-then - void it(Condition firstcond, ItState i1 = kItOmitted, - ItState i2 = kItOmitted, ItState i3 = kItOmitted) OVERRIDE; - - void cbz(Register rn, Label* target) OVERRIDE; - void cbnz(Register rn, Label* target) OVERRIDE; - - // Floating point instructions (VFPv3-D16 and VFPv3-D32 profiles). - void vmovsr(SRegister sn, Register rt, Condition cond = AL) OVERRIDE; - void vmovrs(Register rt, SRegister sn, Condition cond = AL) OVERRIDE; - void vmovsrr(SRegister sm, Register rt, Register rt2, Condition cond = AL) OVERRIDE; - void vmovrrs(Register rt, Register rt2, SRegister sm, Condition cond = AL) OVERRIDE; - void vmovdrr(DRegister dm, Register rt, Register rt2, Condition cond = AL) OVERRIDE; - void vmovrrd(Register rt, Register rt2, DRegister dm, Condition cond = AL) OVERRIDE; - void vmovs(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vmovd(DRegister dd, DRegister dm, Condition cond = AL) OVERRIDE; - - // Returns false if the immediate cannot be encoded. - bool vmovs(SRegister sd, float s_imm, Condition cond = AL) OVERRIDE; - bool vmovd(DRegister dd, double d_imm, Condition cond = AL) OVERRIDE; - - void vldrs(SRegister sd, const Address& ad, Condition cond = AL) OVERRIDE; - void vstrs(SRegister sd, const Address& ad, Condition cond = AL) OVERRIDE; - void vldrd(DRegister dd, const Address& ad, Condition cond = AL) OVERRIDE; - void vstrd(DRegister dd, const Address& ad, Condition cond = AL) OVERRIDE; - - void vadds(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) OVERRIDE; - void vaddd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) OVERRIDE; - void vsubs(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) OVERRIDE; - void vsubd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) OVERRIDE; - void vmuls(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) OVERRIDE; - void vmuld(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) OVERRIDE; - void vmlas(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) OVERRIDE; - void vmlad(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) OVERRIDE; - void vmlss(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) OVERRIDE; - void vmlsd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) OVERRIDE; - void vdivs(SRegister sd, SRegister sn, SRegister sm, Condition cond = AL) OVERRIDE; - void vdivd(DRegister dd, DRegister dn, DRegister dm, Condition cond = AL) OVERRIDE; - - void vabss(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vabsd(DRegister dd, DRegister dm, Condition cond = AL) OVERRIDE; - void vnegs(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vnegd(DRegister dd, DRegister dm, Condition cond = AL) OVERRIDE; - void vsqrts(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vsqrtd(DRegister dd, DRegister dm, Condition cond = AL) OVERRIDE; - - void vcvtsd(SRegister sd, DRegister dm, Condition cond = AL) OVERRIDE; - void vcvtds(DRegister dd, SRegister sm, Condition cond = AL) OVERRIDE; - void vcvtis(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vcvtid(SRegister sd, DRegister dm, Condition cond = AL) OVERRIDE; - void vcvtsi(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vcvtdi(DRegister dd, SRegister sm, Condition cond = AL) OVERRIDE; - void vcvtus(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vcvtud(SRegister sd, DRegister dm, Condition cond = AL) OVERRIDE; - void vcvtsu(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vcvtdu(DRegister dd, SRegister sm, Condition cond = AL) OVERRIDE; - - void vcmps(SRegister sd, SRegister sm, Condition cond = AL) OVERRIDE; - void vcmpd(DRegister dd, DRegister dm, Condition cond = AL) OVERRIDE; - void vcmpsz(SRegister sd, Condition cond = AL) OVERRIDE; - void vcmpdz(DRegister dd, Condition cond = AL) OVERRIDE; - void vmstat(Condition cond = AL) OVERRIDE; // VMRS APSR_nzcv, FPSCR - - void vcntd(DRegister dd, DRegister dm) OVERRIDE; - void vpaddld(DRegister dd, DRegister dm, int32_t size, bool is_unsigned) OVERRIDE; - - void vpushs(SRegister reg, int nregs, Condition cond = AL) OVERRIDE; - void vpushd(DRegister reg, int nregs, Condition cond = AL) OVERRIDE; - void vpops(SRegister reg, int nregs, Condition cond = AL) OVERRIDE; - void vpopd(DRegister reg, int nregs, Condition cond = AL) OVERRIDE; - void vldmiad(Register base_reg, DRegister reg, int nregs, Condition cond = AL) OVERRIDE; - void vstmiad(Register base_reg, DRegister reg, int nregs, Condition cond = AL) OVERRIDE; - - // Branch instructions. - void b(Label* label, Condition cond = AL); - void bl(Label* label, Condition cond = AL); - void blx(Label* label); - void blx(Register rm, Condition cond = AL) OVERRIDE; - void bx(Register rm, Condition cond = AL) OVERRIDE; - - // ADR instruction loading register for branching to the label, including the Thumb mode bit. - void AdrCode(Register rt, Label* label) OVERRIDE; - - virtual void Lsl(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - virtual void Lsr(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - virtual void Asr(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - virtual void Ror(Register rd, Register rm, uint32_t shift_imm, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - virtual void Rrx(Register rd, Register rm, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - virtual void Lsl(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - virtual void Lsr(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - virtual void Asr(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - virtual void Ror(Register rd, Register rm, Register rn, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - void Push(Register rd, Condition cond = AL) OVERRIDE; - void Pop(Register rd, Condition cond = AL) OVERRIDE; - - void PushList(RegList regs, Condition cond = AL) OVERRIDE; - void PopList(RegList regs, Condition cond = AL) OVERRIDE; - void StoreList(RegList regs, size_t stack_offset) OVERRIDE; - void LoadList(RegList regs, size_t stack_offset) OVERRIDE; - - void Mov(Register rd, Register rm, Condition cond = AL) OVERRIDE; - - void CompareAndBranchIfZero(Register r, Label* label) OVERRIDE; - void CompareAndBranchIfNonZero(Register r, Label* label) OVERRIDE; - - // Memory barriers. - void dmb(DmbOptions flavor) OVERRIDE; - - // Get the final position of a label after local fixup based on the old position - // recorded before FinalizeCode(). - uint32_t GetAdjustedPosition(uint32_t old_position) OVERRIDE; - - using ArmAssembler::NewLiteral; // Make the helper template visible. - - Literal* NewLiteral(size_t size, const uint8_t* data) OVERRIDE; - void LoadLiteral(Register rt, Literal* literal) OVERRIDE; - void LoadLiteral(Register rt, Register rt2, Literal* literal) OVERRIDE; - void LoadLiteral(SRegister sd, Literal* literal) OVERRIDE; - void LoadLiteral(DRegister dd, Literal* literal) OVERRIDE; - - // Add signed constant value to rd. May clobber IP. - void AddConstant(Register rd, Register rn, int32_t value, - Condition cond = AL, SetCc set_cc = kCcDontCare) OVERRIDE; - - void CmpConstant(Register rn, int32_t value, Condition cond = AL) OVERRIDE; - - // Load and Store. May clobber IP. - void LoadImmediate(Register rd, int32_t value, Condition cond = AL) OVERRIDE; - void LoadDImmediate(DRegister dd, double value, Condition cond = AL) OVERRIDE; - void MarkExceptionHandler(Label* label) OVERRIDE; - void LoadFromOffset(LoadOperandType type, - Register reg, - Register base, - int32_t offset, - Condition cond = AL) OVERRIDE; - void StoreToOffset(StoreOperandType type, - Register reg, - Register base, - int32_t offset, - Condition cond = AL) OVERRIDE; - void LoadSFromOffset(SRegister reg, - Register base, - int32_t offset, - Condition cond = AL) OVERRIDE; - void StoreSToOffset(SRegister reg, - Register base, - int32_t offset, - Condition cond = AL) OVERRIDE; - void LoadDFromOffset(DRegister reg, - Register base, - int32_t offset, - Condition cond = AL) OVERRIDE; - void StoreDToOffset(DRegister reg, - Register base, - int32_t offset, - Condition cond = AL) OVERRIDE; - - bool ShifterOperandCanHold(Register rd, - Register rn, - Opcode opcode, - uint32_t immediate, - SetCc set_cc, - ShifterOperand* shifter_op) OVERRIDE; - using ArmAssembler::ShifterOperandCanHold; // Don't hide the non-virtual override. - - bool ShifterOperandCanAlwaysHold(uint32_t immediate) OVERRIDE; - - - static bool IsInstructionForExceptionHandling(uintptr_t pc); - - // Emit data (e.g. encoded instruction or immediate) to the. - // instruction stream. - void Emit32(int32_t value); // Emit a 32 bit instruction in thumb format. - void Emit16(int16_t value); // Emit a 16 bit instruction in little endian format. - void Bind(Label* label) OVERRIDE; - - // Force the assembler to generate 32 bit instructions. - void Force32Bit() { - force_32bit_ = true; - } - - void Allow16Bit() { - force_32bit_ = false; - } - - // Emit an ADR (or a sequence of instructions) to load the jump table address into base_reg. This - // will generate a fixup. - JumpTable* CreateJumpTable(std::vector<Label*>&& labels, Register base_reg) OVERRIDE; - // Emit an ADD PC, X to dispatch a jump-table jump. This will generate a fixup. - void EmitJumpTableDispatch(JumpTable* jump_table, Register displacement_reg) OVERRIDE; - - private: - typedef uint16_t FixupId; - - // Fixup: branches and literal pool references. - // - // The thumb2 architecture allows branches to be either 16 or 32 bit instructions. This - // depends on both the type of branch and the offset to which it is branching. The 16-bit - // cbz and cbnz instructions may also need to be replaced with a separate 16-bit compare - // instruction and a 16- or 32-bit branch instruction. Load from a literal pool can also be - // 16-bit or 32-bit instruction and, if the method is large, we may need to use a sequence - // of instructions to make up for the limited range of load literal instructions (up to - // 4KiB for the 32-bit variant). When generating code for these insns we don't know the - // size before hand, so we assume it is the smallest available size and determine the final - // code offsets and sizes and emit code in FinalizeCode(). - // - // To handle this, we keep a record of every branch and literal pool load in the program. - // The actual instruction encoding for these is delayed until we know the final size of - // every instruction. When we bind a label to a branch we don't know the final location yet - // as some preceding instructions may need to be expanded, so we record a non-final offset. - // In FinalizeCode(), we expand the sizes of branches and literal loads that are out of - // range. With each expansion, we need to update dependent Fixups, i.e. insntructios with - // target on the other side of the expanded insn, as their offsets change and this may - // trigger further expansion. - // - // All Fixups have a 'fixup id' which is a 16 bit unsigned number used to identify the - // Fixup. For each unresolved label we keep a singly-linked list of all Fixups pointing - // to it, using the fixup ids as links. The first link is stored in the label's position - // (the label is linked but not bound), the following links are stored in the code buffer, - // in the placeholder where we will eventually emit the actual code. - - class Fixup { - public: - // Branch type. - enum Type : uint8_t { - kConditional, // B<cond>. - kUnconditional, // B. - kUnconditionalLink, // BL. - kUnconditionalLinkX, // BLX. - kCompareAndBranchXZero, // cbz/cbnz. - kLoadCodeAddr, // Get address of a code label, used for Baker read barriers. - kLoadLiteralNarrow, // Load narrrow integer literal. - kLoadLiteralWide, // Load wide integer literal. - kLoadLiteralAddr, // Load address of literal (used for jump table). - kLoadFPLiteralSingle, // Load FP literal single. - kLoadFPLiteralDouble, // Load FP literal double. - }; - - // Calculated size of branch instruction based on type and offset. - enum Size : uint8_t { - // Branch variants. - kBranch16Bit, - kBranch32Bit, - // NOTE: We don't support branches which would require multiple instructions, i.e. - // conditinoal branches beyond +-1MiB and unconditional branches beyond +-16MiB. - - // CBZ/CBNZ variants. - kCbxz16Bit, // CBZ/CBNZ rX, label; X < 8; 7-bit positive offset. - kCbxz32Bit, // CMP rX, #0 + Bcc label; X < 8; 16-bit Bcc; +-8-bit offset. - kCbxz48Bit, // CMP rX, #0 + Bcc label; X < 8; 32-bit Bcc; up to +-1MiB offset. - - // ADR variants. - kCodeAddr4KiB, // ADR rX, <label>; label must be after the ADR but within 4KiB range. - // Multi-instruction expansion is not supported. - - // Load integer literal variants. - // LDR rX, label; X < 8; 16-bit variant up to 1KiB offset; 2 bytes. - kLiteral1KiB, - // LDR rX, label; 32-bit variant up to 4KiB offset; 4 bytes. - kLiteral4KiB, - // MOV rX, imm16 + ADD rX, pc + LDR rX, [rX]; X < 8; up to 64KiB offset; 8 bytes. - kLiteral64KiB, - // MOV rX, modimm + ADD rX, pc + LDR rX, [rX, #imm12]; up to 1MiB offset; 10 bytes. - kLiteral1MiB, - // NOTE: We don't provide the 12-byte version of kLiteralFar below where the LDR is 16-bit. - // MOV rX, imm16 + MOVT rX, imm16 + ADD rX, pc + LDR rX, [rX]; any offset; 14 bytes. - kLiteralFar, - - // Load literal base addr. - // ADR rX, label; X < 8; 8 bit immediate, shifted to 10 bit. 2 bytes. - kLiteralAddr1KiB, - // ADR rX, label; 4KiB offset. 4 bytes. - kLiteralAddr4KiB, - // MOV rX, imm16 + ADD rX, pc; 64KiB offset. 6 bytes. - kLiteralAddr64KiB, - // MOV rX, imm16 + MOVT rX, imm16 + ADD rX, pc; any offset; 10 bytes. - kLiteralAddrFar, - - // Load long or FP literal variants. - // VLDR s/dX, label; 32-bit insn, up to 1KiB offset; 4 bytes. - kLongOrFPLiteral1KiB, - // MOV ip, imm16 + ADD ip, pc + VLDR s/dX, [IP, #0]; up to 64KiB offset; 10 bytes. - kLongOrFPLiteral64KiB, - // MOV ip, imm16 + MOVT ip, imm16 + ADD ip, pc + VLDR s/dX, [IP]; any offset; 14 bytes. - kLongOrFPLiteralFar, - }; - - // Unresolved branch possibly with a condition. - static Fixup Branch(uint32_t location, Type type, Size size = kBranch16Bit, - Condition cond = AL) { - DCHECK(type == kConditional || type == kUnconditional || - type == kUnconditionalLink || type == kUnconditionalLinkX); - DCHECK(size == kBranch16Bit || size == kBranch32Bit); - DCHECK(size == kBranch32Bit || (type == kConditional || type == kUnconditional)); - return Fixup(kNoRegister, kNoRegister, kNoSRegister, kNoDRegister, - cond, type, size, location); - } - - // Unresolved compare-and-branch instruction with a register and condition (EQ or NE). - static Fixup CompareAndBranch(uint32_t location, Register rn, Condition cond) { - DCHECK(cond == EQ || cond == NE); - return Fixup(rn, kNoRegister, kNoSRegister, kNoDRegister, - cond, kCompareAndBranchXZero, kCbxz16Bit, location); - } - - // Code address. - static Fixup LoadCodeAddress(uint32_t location, Register rt) { - return Fixup(rt, kNoRegister, kNoSRegister, kNoDRegister, - AL, kLoadCodeAddr, kCodeAddr4KiB, location); - } - - // Load narrow literal. - static Fixup LoadNarrowLiteral(uint32_t location, Register rt, Size size) { - DCHECK(size == kLiteral1KiB || size == kLiteral4KiB || size == kLiteral64KiB || - size == kLiteral1MiB || size == kLiteralFar); - DCHECK(!IsHighRegister(rt) || (size != kLiteral1KiB && size != kLiteral64KiB)); - return Fixup(rt, kNoRegister, kNoSRegister, kNoDRegister, - AL, kLoadLiteralNarrow, size, location); - } - - // Load wide literal. - static Fixup LoadWideLiteral(uint32_t location, Register rt, Register rt2, - Size size = kLongOrFPLiteral1KiB) { - DCHECK(size == kLongOrFPLiteral1KiB || size == kLongOrFPLiteral64KiB || - size == kLongOrFPLiteralFar); - DCHECK(!IsHighRegister(rt) || (size != kLiteral1KiB && size != kLiteral64KiB)); - return Fixup(rt, rt2, kNoSRegister, kNoDRegister, - AL, kLoadLiteralWide, size, location); - } - - // Load FP single literal. - static Fixup LoadSingleLiteral(uint32_t location, SRegister sd, - Size size = kLongOrFPLiteral1KiB) { - DCHECK(size == kLongOrFPLiteral1KiB || size == kLongOrFPLiteral64KiB || - size == kLongOrFPLiteralFar); - return Fixup(kNoRegister, kNoRegister, sd, kNoDRegister, - AL, kLoadFPLiteralSingle, size, location); - } - - // Load FP double literal. - static Fixup LoadDoubleLiteral(uint32_t location, DRegister dd, - Size size = kLongOrFPLiteral1KiB) { - DCHECK(size == kLongOrFPLiteral1KiB || size == kLongOrFPLiteral64KiB || - size == kLongOrFPLiteralFar); - return Fixup(kNoRegister, kNoRegister, kNoSRegister, dd, - AL, kLoadFPLiteralDouble, size, location); - } - - static Fixup LoadLiteralAddress(uint32_t location, Register rt, Size size) { - DCHECK(size == kLiteralAddr1KiB || size == kLiteralAddr4KiB || size == kLiteralAddr64KiB || - size == kLiteralAddrFar); - DCHECK(!IsHighRegister(rt) || size != kLiteralAddr1KiB); - return Fixup(rt, kNoRegister, kNoSRegister, kNoDRegister, - AL, kLoadLiteralAddr, size, location); - } - - Type GetType() const { - return type_; - } - - bool IsLoadLiteral() const { - return GetType() >= kLoadLiteralNarrow; - } - - // Returns whether the Fixup can expand from the original size. - bool CanExpand() const { - switch (GetOriginalSize()) { - case kBranch32Bit: - case kCbxz48Bit: - case kCodeAddr4KiB: - case kLiteralFar: - case kLiteralAddrFar: - case kLongOrFPLiteralFar: - return false; - default: - return true; - } - } - - Size GetOriginalSize() const { - return original_size_; - } - - Size GetSize() const { - return size_; - } - - uint32_t GetOriginalSizeInBytes() const; - - uint32_t GetSizeInBytes() const; - - uint32_t GetLocation() const { - return location_; - } - - uint32_t GetTarget() const { - return target_; - } - - uint32_t GetAdjustment() const { - return adjustment_; - } - - // Prepare the assembler->fixup_dependents_ and each Fixup's dependents_start_/count_. - static void PrepareDependents(Thumb2Assembler* assembler); - - ArrayRef<const FixupId> Dependents(const Thumb2Assembler& assembler) const { - return ArrayRef<const FixupId>(assembler.fixup_dependents_).SubArray(dependents_start_, - dependents_count_); - } - - // Resolve a branch when the target is known. - void Resolve(uint32_t target) { - DCHECK_EQ(target_, kUnresolved); - DCHECK_NE(target, kUnresolved); - target_ = target; - } - - // Branches with bound targets that are in range can be emitted early. - // However, the caller still needs to check if the branch doesn't go over - // another Fixup that's not ready to be emitted. - bool IsCandidateForEmitEarly() const; - - // Check if the current size is OK for current location_, target_ and adjustment_. - // If not, increase the size. Return the size increase, 0 if unchanged. - // If the target if after this Fixup, also add the difference to adjustment_, - // so that we don't need to consider forward Fixups as their own dependencies. - uint32_t AdjustSizeIfNeeded(uint32_t current_code_size); - - // Increase adjustments. This is called for dependents of a Fixup when its size changes. - void IncreaseAdjustment(uint32_t increase) { - adjustment_ += increase; - } - - // Finalize the branch with an adjustment to the location. Both location and target are updated. - void Finalize(uint32_t location_adjustment) { - DCHECK_NE(target_, kUnresolved); - location_ += location_adjustment; - target_ += location_adjustment; - } - - // Emit the branch instruction into the assembler buffer. This does the - // encoding into the thumb instruction. - void Emit(uint32_t emit_location, AssemblerBuffer* buffer, uint32_t code_size) const; - - private: - Fixup(Register rn, Register rt2, SRegister sd, DRegister dd, - Condition cond, Type type, Size size, uint32_t location) - : rn_(rn), - rt2_(rt2), - sd_(sd), - dd_(dd), - cond_(cond), - type_(type), - original_size_(size), size_(size), - location_(location), - target_(kUnresolved), - adjustment_(0u), - dependents_count_(0u), - dependents_start_(0u) { - } - - static size_t SizeInBytes(Size size); - - // The size of padding added before the literal pool. - static size_t LiteralPoolPaddingSize(uint32_t current_code_size); - - // Returns the offset from the PC-using insn to the target. - int32_t GetOffset(uint32_t current_code_size) const; - - size_t IncreaseSize(Size new_size); - - int32_t LoadWideOrFpEncoding(Register rbase, int32_t offset) const; - - template <typename Function> - static void ForExpandableDependencies(Thumb2Assembler* assembler, Function fn); - - static constexpr uint32_t kUnresolved = 0xffffffff; // Value for target_ for unresolved. - - const Register rn_; // Rn for cbnz/cbz, Rt for literal loads. - Register rt2_; // For kLoadLiteralWide. - SRegister sd_; // For kLoadFPLiteralSingle. - DRegister dd_; // For kLoadFPLiteralDouble. - const Condition cond_; - const Type type_; - Size original_size_; - Size size_; - uint32_t location_; // Offset into assembler buffer in bytes. - uint32_t target_; // Offset into assembler buffer in bytes. - uint32_t adjustment_; // The number of extra bytes inserted between location_ and target_. - // Fixups that require adjustment when current size changes are stored in a single - // array in the assembler and we store only the start index and count here. - uint32_t dependents_count_; - uint32_t dependents_start_; - }; - - // Emit a single 32 or 16 bit data processing instruction. - void EmitDataProcessing(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so); - - // Emit a single 32 bit miscellaneous instruction. - void Emit32Miscellaneous(uint8_t op1, - uint8_t op2, - uint32_t rest_encoding); - - // Emit reverse byte instructions: rev, rev16, revsh. - void EmitReverseBytes(Register rd, Register rm, uint32_t op); - - // Emit a single 16 bit miscellaneous instruction. - void Emit16Miscellaneous(uint32_t rest_encoding); - - // Must the instruction be 32 bits or can it possibly be encoded - // in 16 bits? - bool Is32BitDataProcessing(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so); - - // Emit a 32 bit data processing instruction. - void Emit32BitDataProcessing(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so); - - // Emit a 16 bit data processing instruction. - void Emit16BitDataProcessing(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so); - - void Emit16BitAddSub(Condition cond, - Opcode opcode, - SetCc set_cc, - Register rn, - Register rd, - const ShifterOperand& so); - - uint16_t EmitCompareAndBranch(Register rn, uint16_t prev, bool n); - - void EmitLoadStore(Condition cond, - bool load, - bool byte, - bool half, - bool is_signed, - Register rd, - const Address& ad); - - void EmitMemOpAddressMode3(Condition cond, - int32_t mode, - Register rd, - const Address& ad); - - void EmitMultiMemOp(Condition cond, - BlockAddressMode am, - bool load, - Register base, - RegList regs); - - void EmitMulOp(Condition cond, - int32_t opcode, - Register rd, - Register rn, - Register rm, - Register rs); - - void EmitVFPsss(Condition cond, - int32_t opcode, - SRegister sd, - SRegister sn, - SRegister sm); - - void EmitVLdmOrStm(int32_t rest, - uint32_t reg, - int nregs, - Register rn, - bool is_load, - bool dbl, - Condition cond); - - void EmitVFPddd(Condition cond, - int32_t opcode, - DRegister dd, - DRegister dn, - DRegister dm); - - void EmitVFPsd(Condition cond, - int32_t opcode, - SRegister sd, - DRegister dm); - - void EmitVFPds(Condition cond, - int32_t opcode, - DRegister dd, - SRegister sm); - - void EmitVPushPop(uint32_t reg, int nregs, bool push, bool dbl, Condition cond); - - void EmitBranch(Condition cond, Label* label, bool link, bool x); - static int32_t EncodeBranchOffset(int32_t offset, int32_t inst); - static int DecodeBranchOffset(int32_t inst); - void EmitShift(Register rd, Register rm, Shift shift, uint8_t amount, - Condition cond = AL, SetCc set_cc = kCcDontCare); - void EmitShift(Register rd, Register rn, Shift shift, Register rm, - Condition cond = AL, SetCc set_cc = kCcDontCare); - - static int32_t GetAllowedLoadOffsetBits(LoadOperandType type); - static int32_t GetAllowedStoreOffsetBits(StoreOperandType type); - bool CanSplitLoadStoreOffset(int32_t allowed_offset_bits, - int32_t offset, - /*out*/ int32_t* add_to_base, - /*out*/ int32_t* offset_for_load_store); - int32_t AdjustLoadStoreOffset(int32_t allowed_offset_bits, - Register temp, - Register base, - int32_t offset, - Condition cond); - - // Whether the assembler can relocate branches. If false, unresolved branches will be - // emitted on 32bits. - bool can_relocate_branches_; - - // Force the assembler to use 32 bit thumb2 instructions. - bool force_32bit_; - - // IfThen conditions. Used to check that conditional instructions match the preceding IT. - Condition it_conditions_[4]; - uint8_t it_cond_index_; - Condition next_condition_; - - void SetItCondition(ItState s, Condition cond, uint8_t index); - - void CheckCondition(Condition cond) { - CHECK_EQ(cond, next_condition_); - - // Move to the next condition if there is one. - if (it_cond_index_ < 3) { - ++it_cond_index_; - next_condition_ = it_conditions_[it_cond_index_]; - } else { - next_condition_ = AL; - } - } - - void CheckConditionLastIt(Condition cond) { - if (it_cond_index_ < 3) { - // Check that the next condition is AL. This means that the - // current condition is the last in the IT block. - CHECK_EQ(it_conditions_[it_cond_index_ + 1], AL); - } - CheckCondition(cond); - } - - FixupId AddFixup(Fixup fixup) { - FixupId fixup_id = static_cast<FixupId>(fixups_.size()); - fixups_.push_back(fixup); - // For iterating using FixupId, we need the next id to be representable. - DCHECK_EQ(static_cast<size_t>(static_cast<FixupId>(fixups_.size())), fixups_.size()); - return fixup_id; - } - - Fixup* GetFixup(FixupId fixup_id) { - DCHECK_LT(fixup_id, fixups_.size()); - return &fixups_[fixup_id]; - } - - void BindLabel(Label* label, uint32_t bound_pc); - uint32_t BindLiterals(); - void BindJumpTables(uint32_t code_size); - void AdjustFixupIfNeeded(Fixup* fixup, uint32_t* current_code_size, - std::deque<FixupId>* fixups_to_recalculate); - uint32_t AdjustFixups(); - void EmitFixups(uint32_t adjusted_code_size); - void EmitLiterals(); - void EmitJumpTables(); - void PatchCFI(); - - static int16_t BEncoding16(int32_t offset, Condition cond); - static int32_t BEncoding32(int32_t offset, Condition cond); - static int16_t CbxzEncoding16(Register rn, int32_t offset, Condition cond); - static int16_t CmpRnImm8Encoding16(Register rn, int32_t value); - static int16_t AddRdnRmEncoding16(Register rdn, Register rm); - static int32_t MovwEncoding32(Register rd, int32_t value); - static int32_t MovtEncoding32(Register rd, int32_t value); - static int32_t MovModImmEncoding32(Register rd, int32_t value); - static int16_t LdrLitEncoding16(Register rt, int32_t offset); - static int32_t LdrLitEncoding32(Register rt, int32_t offset); - static int32_t LdrdEncoding32(Register rt, Register rt2, Register rn, int32_t offset); - static int32_t VldrsEncoding32(SRegister sd, Register rn, int32_t offset); - static int32_t VldrdEncoding32(DRegister dd, Register rn, int32_t offset); - static int16_t LdrRtRnImm5Encoding16(Register rt, Register rn, int32_t offset); - static int32_t LdrRtRnImm12Encoding(Register rt, Register rn, int32_t offset); - static int16_t AdrEncoding16(Register rd, int32_t offset); - static int32_t AdrEncoding32(Register rd, int32_t offset); - - ArenaVector<Fixup> fixups_; - ArenaVector<FixupId> fixup_dependents_; - - // Use std::deque<> for literal labels to allow insertions at the end - // without invalidating pointers and references to existing elements. - ArenaDeque<Literal> literals_; - - // Deduplication map for 64-bit literals, used for LoadDImmediate(). - ArenaSafeMap<uint64_t, Literal*> literal64_dedupe_map_; - - // Jump table list. - ArenaDeque<JumpTable> jump_tables_; - - // Data for AdjustedPosition(), see the description there. - uint32_t last_position_adjustment_; - uint32_t last_old_position_; - FixupId last_fixup_id_; -}; - -class ScopedForce32Bit { - public: - explicit ScopedForce32Bit(Thumb2Assembler* assembler, bool force = true) - : assembler_(assembler), old_force_32bit_(assembler->IsForced32Bit()) { - if (force) { - assembler->Force32Bit(); - } - } - - ~ScopedForce32Bit() { - if (!old_force_32bit_) { - assembler_->Allow16Bit(); - } - } - - private: - Thumb2Assembler* const assembler_; - const bool old_force_32bit_; -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_UTILS_ARM_ASSEMBLER_THUMB2_H_ diff --git a/compiler/utils/arm/assembler_thumb2_test.cc b/compiler/utils/arm/assembler_thumb2_test.cc deleted file mode 100644 index 0147a76744..0000000000 --- a/compiler/utils/arm/assembler_thumb2_test.cc +++ /dev/null @@ -1,1666 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "assembler_thumb2.h" - -#include "android-base/stringprintf.h" - -#include "base/stl_util.h" -#include "utils/assembler_test.h" - -namespace art { - -using android::base::StringPrintf; - -class AssemblerThumb2Test : public AssemblerTest<arm::Thumb2Assembler, - arm::Register, arm::SRegister, - uint32_t> { - protected: - std::string GetArchitectureString() OVERRIDE { - return "arm"; - } - - std::string GetAssemblerParameters() OVERRIDE { - return " -march=armv7-a -mcpu=cortex-a15 -mfpu=neon -mthumb"; - } - - const char* GetAssemblyHeader() OVERRIDE { - return kThumb2AssemblyHeader; - } - - std::string GetDisassembleParameters() OVERRIDE { - return " -D -bbinary -marm --disassembler-options=force-thumb --no-show-raw-insn"; - } - - void SetUpHelpers() OVERRIDE { - if (registers_.size() == 0) { - registers_.insert(end(registers_), - { // NOLINT(whitespace/braces) - new arm::Register(arm::R0), - new arm::Register(arm::R1), - new arm::Register(arm::R2), - new arm::Register(arm::R3), - new arm::Register(arm::R4), - new arm::Register(arm::R5), - new arm::Register(arm::R6), - new arm::Register(arm::R7), - new arm::Register(arm::R8), - new arm::Register(arm::R9), - new arm::Register(arm::R10), - new arm::Register(arm::R11), - new arm::Register(arm::R12), - new arm::Register(arm::R13), - new arm::Register(arm::R14), - new arm::Register(arm::R15) - }); - } - } - - void TearDown() OVERRIDE { - AssemblerTest::TearDown(); - STLDeleteElements(®isters_); - } - - std::vector<arm::Register*> GetRegisters() OVERRIDE { - return registers_; - } - - uint32_t CreateImmediate(int64_t imm_value) OVERRIDE { - return imm_value; - } - - std::string RepeatInsn(size_t count, const std::string& insn) { - std::string result; - for (; count != 0u; --count) { - result += insn; - } - return result; - } - - private: - std::vector<arm::Register*> registers_; - - static constexpr const char* kThumb2AssemblyHeader = ".syntax unified\n.thumb\n"; -}; - -TEST_F(AssemblerThumb2Test, Toolchain) { - EXPECT_TRUE(CheckTools()); -} - -#define __ GetAssembler()-> - -TEST_F(AssemblerThumb2Test, Sbfx) { - __ sbfx(arm::R0, arm::R1, 0, 1); - __ sbfx(arm::R0, arm::R1, 0, 8); - __ sbfx(arm::R0, arm::R1, 0, 16); - __ sbfx(arm::R0, arm::R1, 0, 32); - - __ sbfx(arm::R0, arm::R1, 8, 1); - __ sbfx(arm::R0, arm::R1, 8, 8); - __ sbfx(arm::R0, arm::R1, 8, 16); - __ sbfx(arm::R0, arm::R1, 8, 24); - - __ sbfx(arm::R0, arm::R1, 16, 1); - __ sbfx(arm::R0, arm::R1, 16, 8); - __ sbfx(arm::R0, arm::R1, 16, 16); - - __ sbfx(arm::R0, arm::R1, 31, 1); - - const char* expected = - "sbfx r0, r1, #0, #1\n" - "sbfx r0, r1, #0, #8\n" - "sbfx r0, r1, #0, #16\n" - "sbfx r0, r1, #0, #32\n" - - "sbfx r0, r1, #8, #1\n" - "sbfx r0, r1, #8, #8\n" - "sbfx r0, r1, #8, #16\n" - "sbfx r0, r1, #8, #24\n" - - "sbfx r0, r1, #16, #1\n" - "sbfx r0, r1, #16, #8\n" - "sbfx r0, r1, #16, #16\n" - - "sbfx r0, r1, #31, #1\n"; - DriverStr(expected, "sbfx"); -} - -TEST_F(AssemblerThumb2Test, Ubfx) { - __ ubfx(arm::R0, arm::R1, 0, 1); - __ ubfx(arm::R0, arm::R1, 0, 8); - __ ubfx(arm::R0, arm::R1, 0, 16); - __ ubfx(arm::R0, arm::R1, 0, 32); - - __ ubfx(arm::R0, arm::R1, 8, 1); - __ ubfx(arm::R0, arm::R1, 8, 8); - __ ubfx(arm::R0, arm::R1, 8, 16); - __ ubfx(arm::R0, arm::R1, 8, 24); - - __ ubfx(arm::R0, arm::R1, 16, 1); - __ ubfx(arm::R0, arm::R1, 16, 8); - __ ubfx(arm::R0, arm::R1, 16, 16); - - __ ubfx(arm::R0, arm::R1, 31, 1); - - const char* expected = - "ubfx r0, r1, #0, #1\n" - "ubfx r0, r1, #0, #8\n" - "ubfx r0, r1, #0, #16\n" - "ubfx r0, r1, #0, #32\n" - - "ubfx r0, r1, #8, #1\n" - "ubfx r0, r1, #8, #8\n" - "ubfx r0, r1, #8, #16\n" - "ubfx r0, r1, #8, #24\n" - - "ubfx r0, r1, #16, #1\n" - "ubfx r0, r1, #16, #8\n" - "ubfx r0, r1, #16, #16\n" - - "ubfx r0, r1, #31, #1\n"; - DriverStr(expected, "ubfx"); -} - -TEST_F(AssemblerThumb2Test, Vmstat) { - __ vmstat(); - - const char* expected = "vmrs APSR_nzcv, FPSCR\n"; - - DriverStr(expected, "vmrs"); -} - -TEST_F(AssemblerThumb2Test, ldrexd) { - __ ldrexd(arm::R0, arm::R1, arm::R0); - __ ldrexd(arm::R0, arm::R1, arm::R1); - __ ldrexd(arm::R0, arm::R1, arm::R2); - __ ldrexd(arm::R5, arm::R3, arm::R7); - - const char* expected = - "ldrexd r0, r1, [r0]\n" - "ldrexd r0, r1, [r1]\n" - "ldrexd r0, r1, [r2]\n" - "ldrexd r5, r3, [r7]\n"; - DriverStr(expected, "ldrexd"); -} - -TEST_F(AssemblerThumb2Test, strexd) { - __ strexd(arm::R9, arm::R0, arm::R1, arm::R0); - __ strexd(arm::R9, arm::R0, arm::R1, arm::R1); - __ strexd(arm::R9, arm::R0, arm::R1, arm::R2); - __ strexd(arm::R9, arm::R5, arm::R3, arm::R7); - - const char* expected = - "strexd r9, r0, r1, [r0]\n" - "strexd r9, r0, r1, [r1]\n" - "strexd r9, r0, r1, [r2]\n" - "strexd r9, r5, r3, [r7]\n"; - DriverStr(expected, "strexd"); -} - -TEST_F(AssemblerThumb2Test, clrex) { - __ clrex(); - - const char* expected = "clrex\n"; - DriverStr(expected, "clrex"); -} - -TEST_F(AssemblerThumb2Test, LdrdStrd) { - __ ldrd(arm::R0, arm::Address(arm::R2, 8)); - __ ldrd(arm::R0, arm::Address(arm::R12)); - __ strd(arm::R0, arm::Address(arm::R2, 8)); - - const char* expected = - "ldrd r0, r1, [r2, #8]\n" - "ldrd r0, r1, [r12]\n" - "strd r0, r1, [r2, #8]\n"; - DriverStr(expected, "ldrdstrd"); -} - -TEST_F(AssemblerThumb2Test, eor) { - __ eor(arm::R1, arm::R1, arm::ShifterOperand(arm::R0)); - __ eor(arm::R1, arm::R0, arm::ShifterOperand(arm::R1)); - __ eor(arm::R1, arm::R8, arm::ShifterOperand(arm::R0)); - __ eor(arm::R8, arm::R1, arm::ShifterOperand(arm::R0)); - __ eor(arm::R1, arm::R0, arm::ShifterOperand(arm::R8)); - - const char* expected = - "eors r1, r0\n" - "eor r1, r0, r1\n" - "eor r1, r8, r0\n" - "eor r8, r1, r0\n" - "eor r1, r0, r8\n"; - DriverStr(expected, "abs"); -} - -TEST_F(AssemblerThumb2Test, sub) { - __ subs(arm::R1, arm::R0, arm::ShifterOperand(42)); - __ sub(arm::R1, arm::R0, arm::ShifterOperand(42)); - __ subs(arm::R1, arm::R0, arm::ShifterOperand(arm::R2, arm::ASR, 31)); - __ sub(arm::R1, arm::R0, arm::ShifterOperand(arm::R2, arm::ASR, 31)); - - const char* expected = - "subs r1, r0, #42\n" - "sub.w r1, r0, #42\n" - "subs r1, r0, r2, asr #31\n" - "sub r1, r0, r2, asr #31\n"; - DriverStr(expected, "sub"); -} - -TEST_F(AssemblerThumb2Test, add) { - __ adds(arm::R1, arm::R0, arm::ShifterOperand(42)); - __ add(arm::R1, arm::R0, arm::ShifterOperand(42)); - __ adds(arm::R1, arm::R0, arm::ShifterOperand(arm::R2, arm::ASR, 31)); - __ add(arm::R1, arm::R0, arm::ShifterOperand(arm::R2, arm::ASR, 31)); - - const char* expected = - "adds r1, r0, #42\n" - "add.w r1, r0, #42\n" - "adds r1, r0, r2, asr #31\n" - "add r1, r0, r2, asr #31\n"; - DriverStr(expected, "add"); -} - -TEST_F(AssemblerThumb2Test, umull) { - __ umull(arm::R0, arm::R1, arm::R2, arm::R3); - - const char* expected = - "umull r0, r1, r2, r3\n"; - DriverStr(expected, "umull"); -} - -TEST_F(AssemblerThumb2Test, smull) { - __ smull(arm::R0, arm::R1, arm::R2, arm::R3); - - const char* expected = - "smull r0, r1, r2, r3\n"; - DriverStr(expected, "smull"); -} - -TEST_F(AssemblerThumb2Test, LoadByteFromThumbOffset) { - arm::LoadOperandType type = arm::kLoadUnsignedByte; - - __ LoadFromOffset(type, arm::R0, arm::R7, 0); - __ LoadFromOffset(type, arm::R1, arm::R7, 31); - __ LoadFromOffset(type, arm::R2, arm::R7, 32); - __ LoadFromOffset(type, arm::R3, arm::R7, 4095); - __ LoadFromOffset(type, arm::R4, arm::SP, 0); - - const char* expected = - "ldrb r0, [r7, #0]\n" - "ldrb r1, [r7, #31]\n" - "ldrb.w r2, [r7, #32]\n" - "ldrb.w r3, [r7, #4095]\n" - "ldrb.w r4, [sp, #0]\n"; - DriverStr(expected, "LoadByteFromThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, StoreByteToThumbOffset) { - arm::StoreOperandType type = arm::kStoreByte; - - __ StoreToOffset(type, arm::R0, arm::R7, 0); - __ StoreToOffset(type, arm::R1, arm::R7, 31); - __ StoreToOffset(type, arm::R2, arm::R7, 32); - __ StoreToOffset(type, arm::R3, arm::R7, 4095); - __ StoreToOffset(type, arm::R4, arm::SP, 0); - - const char* expected = - "strb r0, [r7, #0]\n" - "strb r1, [r7, #31]\n" - "strb.w r2, [r7, #32]\n" - "strb.w r3, [r7, #4095]\n" - "strb.w r4, [sp, #0]\n"; - DriverStr(expected, "StoreByteToThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, LoadHalfFromThumbOffset) { - arm::LoadOperandType type = arm::kLoadUnsignedHalfword; - - __ LoadFromOffset(type, arm::R0, arm::R7, 0); - __ LoadFromOffset(type, arm::R1, arm::R7, 62); - __ LoadFromOffset(type, arm::R2, arm::R7, 64); - __ LoadFromOffset(type, arm::R3, arm::R7, 4094); - __ LoadFromOffset(type, arm::R4, arm::SP, 0); - __ LoadFromOffset(type, arm::R5, arm::R7, 1); // Unaligned - - const char* expected = - "ldrh r0, [r7, #0]\n" - "ldrh r1, [r7, #62]\n" - "ldrh.w r2, [r7, #64]\n" - "ldrh.w r3, [r7, #4094]\n" - "ldrh.w r4, [sp, #0]\n" - "ldrh.w r5, [r7, #1]\n"; - DriverStr(expected, "LoadHalfFromThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, StoreHalfToThumbOffset) { - arm::StoreOperandType type = arm::kStoreHalfword; - - __ StoreToOffset(type, arm::R0, arm::R7, 0); - __ StoreToOffset(type, arm::R1, arm::R7, 62); - __ StoreToOffset(type, arm::R2, arm::R7, 64); - __ StoreToOffset(type, arm::R3, arm::R7, 4094); - __ StoreToOffset(type, arm::R4, arm::SP, 0); - __ StoreToOffset(type, arm::R5, arm::R7, 1); // Unaligned - - const char* expected = - "strh r0, [r7, #0]\n" - "strh r1, [r7, #62]\n" - "strh.w r2, [r7, #64]\n" - "strh.w r3, [r7, #4094]\n" - "strh.w r4, [sp, #0]\n" - "strh.w r5, [r7, #1]\n"; - DriverStr(expected, "StoreHalfToThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, LoadWordFromSpPlusOffset) { - arm::LoadOperandType type = arm::kLoadWord; - - __ LoadFromOffset(type, arm::R0, arm::SP, 0); - __ LoadFromOffset(type, arm::R1, arm::SP, 124); - __ LoadFromOffset(type, arm::R2, arm::SP, 128); - __ LoadFromOffset(type, arm::R3, arm::SP, 1020); - __ LoadFromOffset(type, arm::R4, arm::SP, 1024); - __ LoadFromOffset(type, arm::R5, arm::SP, 4092); - __ LoadFromOffset(type, arm::R6, arm::SP, 1); // Unaligned - - const char* expected = - "ldr r0, [sp, #0]\n" - "ldr r1, [sp, #124]\n" - "ldr r2, [sp, #128]\n" - "ldr r3, [sp, #1020]\n" - "ldr.w r4, [sp, #1024]\n" - "ldr.w r5, [sp, #4092]\n" - "ldr.w r6, [sp, #1]\n"; - DriverStr(expected, "LoadWordFromSpPlusOffset"); -} - -TEST_F(AssemblerThumb2Test, StoreWordToSpPlusOffset) { - arm::StoreOperandType type = arm::kStoreWord; - - __ StoreToOffset(type, arm::R0, arm::SP, 0); - __ StoreToOffset(type, arm::R1, arm::SP, 124); - __ StoreToOffset(type, arm::R2, arm::SP, 128); - __ StoreToOffset(type, arm::R3, arm::SP, 1020); - __ StoreToOffset(type, arm::R4, arm::SP, 1024); - __ StoreToOffset(type, arm::R5, arm::SP, 4092); - __ StoreToOffset(type, arm::R6, arm::SP, 1); // Unaligned - - const char* expected = - "str r0, [sp, #0]\n" - "str r1, [sp, #124]\n" - "str r2, [sp, #128]\n" - "str r3, [sp, #1020]\n" - "str.w r4, [sp, #1024]\n" - "str.w r5, [sp, #4092]\n" - "str.w r6, [sp, #1]\n"; - DriverStr(expected, "StoreWordToSpPlusOffset"); -} - -TEST_F(AssemblerThumb2Test, LoadWordFromPcPlusOffset) { - arm::LoadOperandType type = arm::kLoadWord; - - __ LoadFromOffset(type, arm::R0, arm::PC, 0); - __ LoadFromOffset(type, arm::R1, arm::PC, 124); - __ LoadFromOffset(type, arm::R2, arm::PC, 128); - __ LoadFromOffset(type, arm::R3, arm::PC, 1020); - __ LoadFromOffset(type, arm::R4, arm::PC, 1024); - __ LoadFromOffset(type, arm::R5, arm::PC, 4092); - __ LoadFromOffset(type, arm::R6, arm::PC, 1); // Unaligned - - const char* expected = - "ldr r0, [pc, #0]\n" - "ldr r1, [pc, #124]\n" - "ldr r2, [pc, #128]\n" - "ldr r3, [pc, #1020]\n" - "ldr.w r4, [pc, #1024]\n" - "ldr.w r5, [pc, #4092]\n" - "ldr.w r6, [pc, #1]\n"; - DriverStr(expected, "LoadWordFromPcPlusOffset"); -} - -TEST_F(AssemblerThumb2Test, StoreWordToThumbOffset) { - arm::StoreOperandType type = arm::kStoreWord; - int32_t offset = 4092; - ASSERT_TRUE(arm::Address::CanHoldStoreOffsetThumb(type, offset)); - - __ StoreToOffset(type, arm::R0, arm::SP, offset); - __ StoreToOffset(type, arm::IP, arm::SP, offset); - __ StoreToOffset(type, arm::IP, arm::R5, offset); - - const char* expected = - "str r0, [sp, #4092]\n" - "str ip, [sp, #4092]\n" - "str ip, [r5, #4092]\n"; - DriverStr(expected, "StoreWordToThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, StoreWordToNonThumbOffset) { - arm::StoreOperandType type = arm::kStoreWord; - int32_t offset = 4096; - ASSERT_FALSE(arm::Address::CanHoldStoreOffsetThumb(type, offset)); - - __ StoreToOffset(type, arm::R0, arm::SP, offset); - __ StoreToOffset(type, arm::IP, arm::SP, offset); - __ StoreToOffset(type, arm::IP, arm::R5, offset); - - const char* expected = - "add.w ip, sp, #4096\n" // AddConstant(ip, sp, 4096) - "str r0, [ip, #0]\n" - - "str r5, [sp, #-4]!\n" // Push(r5) - "add.w r5, sp, #4096\n" // AddConstant(r5, 4100 & ~0xfff) - "str ip, [r5, #4]\n" // StoreToOffset(type, ip, r5, 4100 & 0xfff) - "ldr r5, [sp], #4\n" // Pop(r5) - - "str r6, [sp, #-4]!\n" // Push(r6) - "add.w r6, r5, #4096\n" // AddConstant(r6, r5, 4096 & ~0xfff) - "str ip, [r6, #0]\n" // StoreToOffset(type, ip, r6, 4096 & 0xfff) - "ldr r6, [sp], #4\n"; // Pop(r6) - DriverStr(expected, "StoreWordToNonThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, StoreWordPairToThumbOffset) { - arm::StoreOperandType type = arm::kStoreWordPair; - int32_t offset = 1020; - ASSERT_TRUE(arm::Address::CanHoldStoreOffsetThumb(type, offset)); - - __ StoreToOffset(type, arm::R0, arm::SP, offset); - // We cannot use IP (i.e. R12) as first source register, as it would - // force us to use SP (i.e. R13) as second source register, which - // would have an "unpredictable" effect according to the ARMv7 - // specification (the T1 encoding describes the result as - // UNPREDICTABLE when of the source registers is R13). - // - // So we use (R11, IP) (e.g. (R11, R12)) as source registers in the - // following instructions. - __ StoreToOffset(type, arm::R11, arm::SP, offset); - __ StoreToOffset(type, arm::R11, arm::R5, offset); - - const char* expected = - "strd r0, r1, [sp, #1020]\n" - "strd r11, ip, [sp, #1020]\n" - "strd r11, ip, [r5, #1020]\n"; - DriverStr(expected, "StoreWordPairToThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, StoreWordPairToNonThumbOffset) { - arm::StoreOperandType type = arm::kStoreWordPair; - int32_t offset = 1024; - ASSERT_FALSE(arm::Address::CanHoldStoreOffsetThumb(type, offset)); - - __ StoreToOffset(type, arm::R0, arm::SP, offset); - // Same comment as in AssemblerThumb2Test.StoreWordPairToThumbOffset - // regarding the use of (R11, IP) (e.g. (R11, R12)) as source - // registers in the following instructions. - __ StoreToOffset(type, arm::R11, arm::SP, offset); - __ StoreToOffset(type, arm::R11, arm::R5, offset); - - const char* expected = - "add.w ip, sp, #1024\n" // AddConstant(ip, sp, 1024) - "strd r0, r1, [ip, #0]\n" - - "str r5, [sp, #-4]!\n" // Push(r5) - "add.w r5, sp, #1024\n" // AddConstant(r5, sp, (1024 + kRegisterSize) & ~0x3fc) - "strd r11, ip, [r5, #4]\n" // StoreToOffset(type, r11, sp, (1024 + kRegisterSize) & 0x3fc) - "ldr r5, [sp], #4\n" // Pop(r5) - - "str r6, [sp, #-4]!\n" // Push(r6) - "add.w r6, r5, #1024\n" // AddConstant(r6, r5, 1024 & ~0x3fc) - "strd r11, ip, [r6, #0]\n" // StoreToOffset(type, r11, r6, 1024 & 0x3fc) - "ldr r6, [sp], #4\n"; // Pop(r6) - DriverStr(expected, "StoreWordPairToNonThumbOffset"); -} - -TEST_F(AssemblerThumb2Test, DistantBackBranch) { - Label start, end; - __ Bind(&start); - constexpr size_t kLdrR0R0Count1 = 256; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ b(&end, arm::EQ); - __ b(&start, arm::LT); - constexpr size_t kLdrR0R0Count2 = 256; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&end); - - std::string expected = - "0:\n" + - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "beq 1f\n" - "blt 0b\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "1:\n"; - DriverStr(expected, "DistantBackBranch"); -} - -TEST_F(AssemblerThumb2Test, TwoCbzMaxOffset) { - Label label0, label1, label2; - __ cbz(arm::R0, &label1); - constexpr size_t kLdrR0R0Count1 = 63; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label0); - __ cbz(arm::R0, &label2); - __ Bind(&label1); - constexpr size_t kLdrR0R0Count2 = 64; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label2); - - std::string expected = - "cbz r0, 1f\n" + // cbz r0, label1 - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "0:\n" - "cbz r0, 2f\n" // cbz r0, label2 - "1:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "2:\n"; - DriverStr(expected, "TwoCbzMaxOffset"); - - EXPECT_EQ(static_cast<uint32_t>(label0.Position()) + 0u, - __ GetAdjustedPosition(label0.Position())); - EXPECT_EQ(static_cast<uint32_t>(label1.Position()) + 0u, - __ GetAdjustedPosition(label1.Position())); - EXPECT_EQ(static_cast<uint32_t>(label2.Position()) + 0u, - __ GetAdjustedPosition(label2.Position())); -} - -TEST_F(AssemblerThumb2Test, TwoCbzBeyondMaxOffset) { - Label label0, label1, label2; - __ cbz(arm::R0, &label1); - constexpr size_t kLdrR0R0Count1 = 63; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label0); - __ cbz(arm::R0, &label2); - __ Bind(&label1); - constexpr size_t kLdrR0R0Count2 = 65; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label2); - - std::string expected = - "cmp r0, #0\n" // cbz r0, label1 - "beq.n 1f\n" + - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "0:\n" - "cmp r0, #0\n" // cbz r0, label2 - "beq.n 2f\n" - "1:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "2:\n"; - DriverStr(expected, "TwoCbzBeyondMaxOffset"); - - EXPECT_EQ(static_cast<uint32_t>(label0.Position()) + 2u, - __ GetAdjustedPosition(label0.Position())); - EXPECT_EQ(static_cast<uint32_t>(label1.Position()) + 4u, - __ GetAdjustedPosition(label1.Position())); - EXPECT_EQ(static_cast<uint32_t>(label2.Position()) + 4u, - __ GetAdjustedPosition(label2.Position())); -} - -TEST_F(AssemblerThumb2Test, TwoCbzSecondAtMaxB16Offset) { - Label label0, label1, label2; - __ cbz(arm::R0, &label1); - constexpr size_t kLdrR0R0Count1 = 62; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label0); - __ cbz(arm::R0, &label2); - __ Bind(&label1); - constexpr size_t kLdrR0R0Count2 = 128; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label2); - - std::string expected = - "cbz r0, 1f\n" + // cbz r0, label1 - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "0:\n" - "cmp r0, #0\n" // cbz r0, label2 - "beq.n 2f\n" - "1:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "2:\n"; - DriverStr(expected, "TwoCbzSecondAtMaxB16Offset"); - - EXPECT_EQ(static_cast<uint32_t>(label0.Position()) + 0u, - __ GetAdjustedPosition(label0.Position())); - EXPECT_EQ(static_cast<uint32_t>(label1.Position()) + 2u, - __ GetAdjustedPosition(label1.Position())); - EXPECT_EQ(static_cast<uint32_t>(label2.Position()) + 2u, - __ GetAdjustedPosition(label2.Position())); -} - -TEST_F(AssemblerThumb2Test, TwoCbzSecondBeyondMaxB16Offset) { - Label label0, label1, label2; - __ cbz(arm::R0, &label1); - constexpr size_t kLdrR0R0Count1 = 62; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label0); - __ cbz(arm::R0, &label2); - __ Bind(&label1); - constexpr size_t kLdrR0R0Count2 = 129; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label2); - - std::string expected = - "cmp r0, #0\n" // cbz r0, label1 - "beq.n 1f\n" + - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "0:\n" - "cmp r0, #0\n" // cbz r0, label2 - "beq.w 2f\n" - "1:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "2:\n"; - DriverStr(expected, "TwoCbzSecondBeyondMaxB16Offset"); - - EXPECT_EQ(static_cast<uint32_t>(label0.Position()) + 2u, - __ GetAdjustedPosition(label0.Position())); - EXPECT_EQ(static_cast<uint32_t>(label1.Position()) + 6u, - __ GetAdjustedPosition(label1.Position())); - EXPECT_EQ(static_cast<uint32_t>(label2.Position()) + 6u, - __ GetAdjustedPosition(label2.Position())); -} - -TEST_F(AssemblerThumb2Test, TwoCbzFirstAtMaxB16Offset) { - Label label0, label1, label2; - __ cbz(arm::R0, &label1); - constexpr size_t kLdrR0R0Count1 = 127; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label0); - __ cbz(arm::R0, &label2); - __ Bind(&label1); - constexpr size_t kLdrR0R0Count2 = 64; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label2); - - std::string expected = - "cmp r0, #0\n" // cbz r0, label1 - "beq.n 1f\n" + - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "0:\n" - "cbz r0, 2f\n" // cbz r0, label2 - "1:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "2:\n"; - DriverStr(expected, "TwoCbzFirstAtMaxB16Offset"); - - EXPECT_EQ(static_cast<uint32_t>(label0.Position()) + 2u, - __ GetAdjustedPosition(label0.Position())); - EXPECT_EQ(static_cast<uint32_t>(label1.Position()) + 2u, - __ GetAdjustedPosition(label1.Position())); - EXPECT_EQ(static_cast<uint32_t>(label2.Position()) + 2u, - __ GetAdjustedPosition(label2.Position())); -} - -TEST_F(AssemblerThumb2Test, TwoCbzFirstBeyondMaxB16Offset) { - Label label0, label1, label2; - __ cbz(arm::R0, &label1); - constexpr size_t kLdrR0R0Count1 = 127; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label0); - __ cbz(arm::R0, &label2); - __ Bind(&label1); - constexpr size_t kLdrR0R0Count2 = 65; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label2); - - std::string expected = - "cmp r0, #0\n" // cbz r0, label1 - "beq.w 1f\n" + - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "0:\n" - "cmp r0, #0\n" // cbz r0, label2 - "beq.n 2f\n" - "1:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "2:\n"; - DriverStr(expected, "TwoCbzFirstBeyondMaxB16Offset"); - - EXPECT_EQ(static_cast<uint32_t>(label0.Position()) + 4u, - __ GetAdjustedPosition(label0.Position())); - EXPECT_EQ(static_cast<uint32_t>(label1.Position()) + 6u, - __ GetAdjustedPosition(label1.Position())); - EXPECT_EQ(static_cast<uint32_t>(label2.Position()) + 6u, - __ GetAdjustedPosition(label2.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralMax1KiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R0, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = 511; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "1:\n" - "ldr.n r0, [pc, #((2f - 1b - 2) & ~2)]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralMax1KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 0u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralBeyondMax1KiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R0, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = 512; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "1:\n" - "ldr.w r0, [pc, #((2f - 1b - 2) & ~2)]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralBeyondMax1KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 2u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralMax4KiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R1, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = 2046; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "1:\n" - "ldr.w r1, [pc, #((2f - 1b - 2) & ~2)]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralMax4KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 2u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralBeyondMax4KiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R1, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = 2047; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "movw r1, #4096\n" // "as" does not consider (2f - 1f - 4) a constant expression for movw. - "1:\n" - "add r1, pc\n" - "ldr r1, [r1, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralBeyondMax4KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 6u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralMax64KiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R1, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1u << 15) - 2u; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "movw r1, #0xfffc\n" // "as" does not consider (2f - 1f - 4) a constant expression for movw. - "1:\n" - "add r1, pc\n" - "ldr r1, [r1, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralMax64KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 6u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralBeyondMax64KiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R1, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1u << 15) - 1u; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "mov.w r1, #((2f - 1f - 4) & ~0xfff)\n" - "1:\n" - "add r1, pc\n" - "ldr r1, [r1, #((2f - 1b - 4) & 0xfff)]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralBeyondMax64KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 8u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralMax1MiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R1, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1u << 19) - 3u; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "mov.w r1, #((2f - 1f - 4) & ~0xfff)\n" - "1:\n" - "add r1, pc\n" - "ldr r1, [r1, #((2f - 1b - 4) & 0xfff)]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralMax1MiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 8u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralBeyondMax1MiB) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R1, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1u << 19) - 2u; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - // "as" does not consider ((2f - 1f - 4) & 0xffff) a constant expression for movw. - "movw r1, #(0x100000 & 0xffff)\n" - // "as" does not consider ((2f - 1f - 4) >> 16) a constant expression for movt. - "movt r1, #(0x100000 >> 16)\n" - "1:\n" - "add r1, pc\n" - "ldr.w r1, [r1, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralBeyondMax1MiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 12u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralFar) { - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R1, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1u << 19) - 2u + 0x1234; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - // "as" does not consider ((2f - 1f - 4) & 0xffff) a constant expression for movw. - "movw r1, #((0x100000 + 2 * 0x1234) & 0xffff)\n" - // "as" does not consider ((2f - 1f - 4) >> 16) a constant expression for movt. - "movt r1, #((0x100000 + 2 * 0x1234) >> 16)\n" - "1:\n" - "add r1, pc\n" - "ldr.w r1, [r1, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralFar"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 12u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralWideMax1KiB) { - arm::Literal* literal = __ NewLiteral<int64_t>(INT64_C(0x1234567887654321)); - __ LoadLiteral(arm::R1, arm::R3, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = 510; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "1:\n" - "ldrd r1, r3, [pc, #((2f - 1b - 2) & ~2)]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x87654321\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralWideMax1KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 0u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralWideBeyondMax1KiB) { - arm::Literal* literal = __ NewLiteral<int64_t>(INT64_C(0x1234567887654321)); - __ LoadLiteral(arm::R1, arm::R3, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = 511; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - // "as" does not consider ((2f - 1f - 4) & 0xffff) a constant expression for movw. - "movw ip, #(0x408 - 0x4 - 4)\n" - "1:\n" - "add ip, pc\n" - "ldrd r1, r3, [ip, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x87654321\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralWideBeyondMax1KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 6u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralSingleMax64KiB) { - // The literal size must match but the type doesn't, so use an int32_t rather than float. - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::S3, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1 << 15) - 3u; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - // "as" does not consider ((2f - 1f - 4) & 0xffff) a constant expression for movw. - "movw ip, #(0x10004 - 0x4 - 4)\n" - "1:\n" - "add ip, pc\n" - "vldr s3, [ip, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralSingleMax64KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 6u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralSingleMax64KiB_UnalignedPC) { - // The literal size must match but the type doesn't, so use an int32_t rather than float. - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ ldr(arm::R0, arm::Address(arm::R0)); - __ LoadLiteral(arm::S3, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1 << 15) - 4u; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - "ldr r0, [r0]\n" - // "as" does not consider ((2f - 1f - 4) & 0xffff) a constant expression for movw. - "movw ip, #(0x10004 - 0x6 - 4)\n" - "1:\n" - "add ip, pc\n" - "vldr s3, [ip, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralSingleMax64KiB_UnalignedPC"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 6u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralDoubleBeyondMax64KiB) { - // The literal size must match but the type doesn't, so use an int64_t rather than double. - arm::Literal* literal = __ NewLiteral<int64_t>(INT64_C(0x1234567887654321)); - __ LoadLiteral(arm::D3, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1 << 15) - 2u; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - // "as" does not consider ((2f - 1f - 4) & 0xffff) a constant expression for movw. - "movw ip, #((0x1000c - 0x8 - 4) & 0xffff)\n" - // "as" does not consider ((2f - 1f - 4) >> 16) a constant expression for movt. - "movt ip, #((0x1000c - 0x8 - 4) >> 16)\n" - "1:\n" - "add ip, pc\n" - "vldr d3, [ip, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x87654321\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralDoubleBeyondMax64KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 10u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralDoubleFar) { - // The literal size must match but the type doesn't, so use an int64_t rather than double. - arm::Literal* literal = __ NewLiteral<int64_t>(INT64_C(0x1234567887654321)); - __ LoadLiteral(arm::D3, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = (1 << 15) - 2u + 0x1234; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - // "as" does not consider ((2f - 1f - 4) & 0xffff) a constant expression for movw. - "movw ip, #((0x1000c + 2 * 0x1234 - 0x8 - 4) & 0xffff)\n" - // "as" does not consider ((2f - 1f - 4) >> 16) a constant expression for movt. - "movt ip, #((0x1000c + 2 * 0x1234 - 0x8 - 4) >> 16)\n" - "1:\n" - "add ip, pc\n" - "vldr d3, [ip, #0]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x87654321\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralDoubleFar"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 10u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, LoadLiteralBeyondMax1KiBDueToAlignmentOnSecondPass) { - // First part: as TwoCbzBeyondMaxOffset but add one 16-bit instruction to the end, - // so that the size is not Aligned<4>(.). On the first pass, the assembler resizes - // the second CBZ because it's out of range, then it will resize the first CBZ - // which has been pushed out of range. Thus, after the first pass, the code size - // will appear Aligned<4>(.) but the final size will not be. - Label label0, label1, label2; - __ cbz(arm::R0, &label1); - constexpr size_t kLdrR0R0Count1 = 63; - for (size_t i = 0; i != kLdrR0R0Count1; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label0); - __ cbz(arm::R0, &label2); - __ Bind(&label1); - constexpr size_t kLdrR0R0Count2 = 65; - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ Bind(&label2); - __ ldr(arm::R0, arm::Address(arm::R0)); - - std::string expected_part1 = - "cmp r0, #0\n" // cbz r0, label1 - "beq.n 1f\n" + - RepeatInsn(kLdrR0R0Count1, "ldr r0, [r0]\n") + - "0:\n" - "cmp r0, #0\n" // cbz r0, label2 - "beq.n 2f\n" - "1:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - "2:\n" // Here the offset is Aligned<4>(.). - "ldr r0, [r0]\n"; // Make the first part - - // Second part: as LoadLiteralMax1KiB with the caveat that the offset of the load - // literal will not be Aligned<4>(.) but it will appear to be when we process the - // instruction during the first pass, so the literal will need a padding and it - // will push the literal out of range, so we shall end up with "ldr.w". - arm::Literal* literal = __ NewLiteral<int32_t>(0x12345678); - __ LoadLiteral(arm::R0, literal); - Label label; - __ Bind(&label); - constexpr size_t kLdrR0R0Count = 511; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - std::string expected = - expected_part1 + - "1:\n" - "ldr.w r0, [pc, #((2f - 1b - 2) & ~2)]\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2, 0\n" - "2:\n" - ".word 0x12345678\n"; - DriverStr(expected, "LoadLiteralMax1KiB"); - - EXPECT_EQ(static_cast<uint32_t>(label.Position()) + 6u, - __ GetAdjustedPosition(label.Position())); -} - -TEST_F(AssemblerThumb2Test, BindTrackedLabel) { - Label non_tracked, tracked, branch_target; - - // A few dummy loads on entry. - constexpr size_t kLdrR0R0Count = 5; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - // A branch that will need to be fixed up. - __ cbz(arm::R0, &branch_target); - - // Some more dummy loads. - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - // Now insert tracked and untracked label. - __ Bind(&non_tracked); - __ BindTrackedLabel(&tracked); - - // A lot of dummy loads, to ensure the branch needs resizing. - constexpr size_t kLdrR0R0CountLong = 60; - for (size_t i = 0; i != kLdrR0R0CountLong; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - // Bind the branch target. - __ Bind(&branch_target); - - // One more load. - __ ldr(arm::R0, arm::Address(arm::R0)); - - std::string expected = - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - "cmp r0, #0\n" // cbz r0, 1f - "beq.n 1f\n" + - RepeatInsn(kLdrR0R0Count + kLdrR0R0CountLong, "ldr r0, [r0]\n") + - "1:\n" - "ldr r0, [r0]\n"; - DriverStr(expected, "BindTrackedLabel"); - - // Expectation is that the tracked label should have moved. - EXPECT_LT(non_tracked.Position(), tracked.Position()); -} - -TEST_F(AssemblerThumb2Test, JumpTable) { - // The jump table. Use three labels. - Label label1, label2, label3; - std::vector<Label*> labels({ &label1, &label2, &label3 }); - - // A few dummy loads on entry, interspersed with 2 labels. - constexpr size_t kLdrR0R0Count = 5; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label1); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label2); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - // Create the jump table, emit the base load. - arm::JumpTable* jump_table = __ CreateJumpTable(std::move(labels), arm::R1); - - // Dummy computation, stand-in for the address. We're only testing the jump table here, not how - // it's being used. - __ ldr(arm::R0, arm::Address(arm::R0)); - - // Emit the jump - __ EmitJumpTableDispatch(jump_table, arm::R1); - - // Some more dummy instructions. - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label3); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { // Note: odd so there's no alignment - __ ldr(arm::R0, arm::Address(arm::R0)); // necessary, as gcc as emits nops, - } // whereas we emit 0 != nop. - - static_assert((kLdrR0R0Count + 3) * 2 < 1 * KB, "Too much offset"); - - std::string expected = - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L1:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L2:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - "adr r1, .Ljump_table\n" - "ldr r0, [r0]\n" - ".Lbase:\n" - "add pc, r1\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L3:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".align 2\n" - ".Ljump_table:\n" - ".4byte (.L1 - .Lbase - 4)\n" - ".4byte (.L2 - .Lbase - 4)\n" - ".4byte (.L3 - .Lbase - 4)\n"; - DriverStr(expected, "JumpTable"); -} - -// Test for >1K fixup. -TEST_F(AssemblerThumb2Test, JumpTable4K) { - // The jump table. Use three labels. - Label label1, label2, label3; - std::vector<Label*> labels({ &label1, &label2, &label3 }); - - // A few dummy loads on entry, interspersed with 2 labels. - constexpr size_t kLdrR0R0Count = 5; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label1); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label2); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - // Create the jump table, emit the base load. - arm::JumpTable* jump_table = __ CreateJumpTable(std::move(labels), arm::R1); - - // Dummy computation, stand-in for the address. We're only testing the jump table here, not how - // it's being used. - __ ldr(arm::R0, arm::Address(arm::R0)); - - // Emit the jump - __ EmitJumpTableDispatch(jump_table, arm::R1); - - // Some more dummy instructions. - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label3); - constexpr size_t kLdrR0R0Count2 = 600; // Note: even so there's no alignment - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { // necessary, as gcc as emits nops, - __ ldr(arm::R0, arm::Address(arm::R0)); // whereas we emit 0 != nop. - } - - static_assert((kLdrR0R0Count + kLdrR0R0Count2 + 3) * 2 > 1 * KB, "Not enough offset"); - static_assert((kLdrR0R0Count + kLdrR0R0Count2 + 3) * 2 < 4 * KB, "Too much offset"); - - std::string expected = - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L1:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L2:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - "adr r1, .Ljump_table\n" - "ldr r0, [r0]\n" - ".Lbase:\n" - "add pc, r1\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L3:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - ".align 2\n" - ".Ljump_table:\n" - ".4byte (.L1 - .Lbase - 4)\n" - ".4byte (.L2 - .Lbase - 4)\n" - ".4byte (.L3 - .Lbase - 4)\n"; - DriverStr(expected, "JumpTable4K"); -} - -// Test for >4K fixup. -TEST_F(AssemblerThumb2Test, JumpTable64K) { - // The jump table. Use three labels. - Label label1, label2, label3; - std::vector<Label*> labels({ &label1, &label2, &label3 }); - - // A few dummy loads on entry, interspersed with 2 labels. - constexpr size_t kLdrR0R0Count = 5; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label1); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label2); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - // Create the jump table, emit the base load. - arm::JumpTable* jump_table = __ CreateJumpTable(std::move(labels), arm::R1); - - // Dummy computation, stand-in for the address. We're only testing the jump table here, not how - // it's being used. - __ ldr(arm::R0, arm::Address(arm::R0)); - - // Emit the jump - __ EmitJumpTableDispatch(jump_table, arm::R1); - - // Some more dummy instructions. - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label3); - constexpr size_t kLdrR0R0Count2 = 2601; // Note: odd so there's no alignment - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { // necessary, as gcc as emits nops, - __ ldr(arm::R0, arm::Address(arm::R0)); // whereas we emit 0 != nop. - } - - static_assert((kLdrR0R0Count + kLdrR0R0Count2 + 3) * 2 > 4 * KB, "Not enough offset"); - static_assert((kLdrR0R0Count + kLdrR0R0Count2 + 3) * 2 < 64 * KB, "Too much offset"); - - std::string expected = - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L1:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L2:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - // ~ adr r1, .Ljump_table, gcc as can't seem to fix up a large offset itself. - // (Note: have to use constants, as labels aren't accepted. - "movw r1, #(((3 + " + StringPrintf("%zu", kLdrR0R0Count + kLdrR0R0Count2) + - ") * 2 - 4) & 0xFFFF)\n" - "add r1, pc\n" - "ldr r0, [r0]\n" - ".Lbase:\n" - "add pc, r1\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L3:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - ".align 2\n" - ".Ljump_table:\n" - ".4byte (.L1 - .Lbase - 4)\n" - ".4byte (.L2 - .Lbase - 4)\n" - ".4byte (.L3 - .Lbase - 4)\n"; - DriverStr(expected, "JumpTable64K"); -} - -// Test for >64K fixup. -TEST_F(AssemblerThumb2Test, JumpTableFar) { - // The jump table. Use three labels. - Label label1, label2, label3; - std::vector<Label*> labels({ &label1, &label2, &label3 }); - - // A few dummy loads on entry, interspersed with 2 labels. - constexpr size_t kLdrR0R0Count = 5; - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label1); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label2); - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - - // Create the jump table, emit the base load. - arm::JumpTable* jump_table = __ CreateJumpTable(std::move(labels), arm::R1); - - // Dummy computation, stand-in for the address. We're only testing the jump table here, not how - // it's being used. - __ ldr(arm::R0, arm::Address(arm::R0)); - - // Emit the jump - __ EmitJumpTableDispatch(jump_table, arm::R1); - - // Some more dummy instructions. - for (size_t i = 0; i != kLdrR0R0Count; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } - __ BindTrackedLabel(&label3); - constexpr size_t kLdrR0R0Count2 = 70001; // Note: odd so there's no alignment - for (size_t i = 0; i != kLdrR0R0Count2; ++i) { // necessary, as gcc as emits nops, - __ ldr(arm::R0, arm::Address(arm::R0)); // whereas we emit 0 != nop. - } - - static_assert((kLdrR0R0Count + kLdrR0R0Count2 + 3) * 2 > 64 * KB, "Not enough offset"); - - std::string expected = - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L1:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L2:\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - // ~ adr r1, .Ljump_table, gcc as can't seem to fix up a large offset itself. - // (Note: have to use constants, as labels aren't accepted. - "movw r1, #(((3 + " + StringPrintf("%zu", kLdrR0R0Count + kLdrR0R0Count2) + - ") * 2 - 4) & 0xFFFF)\n" - "movt r1, #(((3 + " + StringPrintf("%zu", kLdrR0R0Count + kLdrR0R0Count2) + - ") * 2 - 4) >> 16)\n" - ".Lhelp:" - "add r1, pc\n" - "ldr r0, [r0]\n" - ".Lbase:\n" - "add pc, r1\n" + - RepeatInsn(kLdrR0R0Count, "ldr r0, [r0]\n") + - ".L3:\n" + - RepeatInsn(kLdrR0R0Count2, "ldr r0, [r0]\n") + - ".align 2\n" - ".Ljump_table:\n" - ".4byte (.L1 - .Lbase - 4)\n" - ".4byte (.L2 - .Lbase - 4)\n" - ".4byte (.L3 - .Lbase - 4)\n"; - DriverStr(expected, "JumpTableFar"); -} - -TEST_F(AssemblerThumb2Test, Clz) { - __ clz(arm::R0, arm::R1); - - const char* expected = "clz r0, r1\n"; - - DriverStr(expected, "clz"); -} - -TEST_F(AssemblerThumb2Test, rbit) { - __ rbit(arm::R1, arm::R0); - - const char* expected = "rbit r1, r0\n"; - - DriverStr(expected, "rbit"); -} - -TEST_F(AssemblerThumb2Test, rev) { - __ rev(arm::R1, arm::R0); - - const char* expected = "rev r1, r0\n"; - - DriverStr(expected, "rev"); -} - -TEST_F(AssemblerThumb2Test, rev16) { - __ rev16(arm::R1, arm::R0); - - const char* expected = "rev16 r1, r0\n"; - - DriverStr(expected, "rev16"); -} - -TEST_F(AssemblerThumb2Test, revsh) { - __ revsh(arm::R1, arm::R0); - - const char* expected = "revsh r1, r0\n"; - - DriverStr(expected, "revsh"); -} - -TEST_F(AssemblerThumb2Test, vcnt) { - // Different D register numbers are used here, to test register encoding. - // Source register number is encoded as M:Vm, destination register number is encoded as D:Vd, - // For source and destination registers which use D0..D15, the M bit and D bit should be 0. - // For source and destination registers which use D16..D32, the M bit and D bit should be 1. - __ vcntd(arm::D0, arm::D1); - __ vcntd(arm::D19, arm::D20); - __ vcntd(arm::D0, arm::D9); - __ vcntd(arm::D16, arm::D20); - - std::string expected = - "vcnt.8 d0, d1\n" - "vcnt.8 d19, d20\n" - "vcnt.8 d0, d9\n" - "vcnt.8 d16, d20\n"; - - DriverStr(expected, "vcnt"); -} - -TEST_F(AssemblerThumb2Test, vpaddl) { - // Different D register numbers are used here, to test register encoding. - // Source register number is encoded as M:Vm, destination register number is encoded as D:Vd, - // For source and destination registers which use D0..D15, the M bit and D bit should be 0. - // For source and destination registers which use D16..D32, the M bit and D bit should be 1. - // Different data types (signed and unsigned) are also tested. - __ vpaddld(arm::D0, arm::D0, 8, true); - __ vpaddld(arm::D20, arm::D20, 8, false); - __ vpaddld(arm::D0, arm::D20, 16, false); - __ vpaddld(arm::D20, arm::D0, 32, true); - - std::string expected = - "vpaddl.u8 d0, d0\n" - "vpaddl.s8 d20, d20\n" - "vpaddl.s16 d0, d20\n" - "vpaddl.u32 d20, d0\n"; - - DriverStr(expected, "vpaddl"); -} - -TEST_F(AssemblerThumb2Test, LoadFromShiftedRegOffset) { - arm::Address mem_address(arm::R0, arm::R1, arm::Shift::LSL, 2); - - __ ldrsb(arm::R2, mem_address); - __ ldrb(arm::R2, mem_address); - __ ldrsh(arm::R2, mem_address); - __ ldrh(arm::R2, mem_address); - __ ldr(arm::R2, mem_address); - - std::string expected = - "ldrsb r2, [r0, r1, LSL #2]\n" - "ldrb r2, [r0, r1, LSL #2]\n" - "ldrsh r2, [r0, r1, LSL #2]\n" - "ldrh r2, [r0, r1, LSL #2]\n" - "ldr r2, [r0, r1, LSL #2]\n"; - - DriverStr(expected, "LoadFromShiftedRegOffset"); -} - -TEST_F(AssemblerThumb2Test, VStmLdmPushPop) { - // Different D register numbers are used here, to test register encoding. - // Source register number is encoded as M:Vm, destination register number is encoded as D:Vd, - // For source and destination registers which use D0..D15, the M bit and D bit should be 0. - // For source and destination registers which use D16..D32, the M bit and D bit should be 1. - // Different data types (signed and unsigned) are also tested. - __ vstmiad(arm::R0, arm::D0, 4); - __ vldmiad(arm::R1, arm::D9, 5); - __ vpopd(arm::D0, 4); - __ vpushd(arm::D9, 5); - __ vpops(arm::S0, 4); - __ vpushs(arm::S9, 5); - __ vpushs(arm::S16, 5); - __ vpushd(arm::D0, 16); - __ vpushd(arm::D1, 15); - __ vpushd(arm::D8, 16); - __ vpushd(arm::D31, 1); - __ vpushs(arm::S0, 32); - __ vpushs(arm::S1, 31); - __ vpushs(arm::S16, 16); - __ vpushs(arm::S31, 1); - - std::string expected = - "vstmia r0, {d0 - d3}\n" - "vldmia r1, {d9 - d13}\n" - "vpop {d0 - d3}\n" - "vpush {d9 - d13}\n" - "vpop {s0 - s3}\n" - "vpush {s9 - s13}\n" - "vpush {s16 - s20}\n" - "vpush {d0 - d15}\n" - "vpush {d1 - d15}\n" - "vpush {d8 - d23}\n" - "vpush {d31}\n" - "vpush {s0 - s31}\n" - "vpush {s1 - s31}\n" - "vpush {s16 - s31}\n" - "vpush {s31}\n"; - - DriverStr(expected, "VStmLdmPushPop"); -} - -} // namespace art diff --git a/compiler/utils/arm/constants_arm.cc b/compiler/utils/arm/constants_arm.cc new file mode 100644 index 0000000000..b02b343b26 --- /dev/null +++ b/compiler/utils/arm/constants_arm.cc @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "constants_arm.h" + +namespace art { +namespace arm { + +std::ostream& operator<<(std::ostream& os, const DRegister& rhs) { + if (rhs >= D0 && rhs < kNumberOfDRegisters) { + os << "d" << static_cast<int>(rhs); + } else { + os << "DRegister[" << static_cast<int>(rhs) << "]"; + } + return os; +} + +} // namespace arm +} // namespace art diff --git a/compiler/utils/arm/constants_arm.h b/compiler/utils/arm/constants_arm.h index 2060064423..5b87e3e7f8 100644 --- a/compiler/utils/arm/constants_arm.h +++ b/compiler/utils/arm/constants_arm.h @@ -97,37 +97,6 @@ enum DRegister { // private marker to avoid generate-operator-out.py from proce }; std::ostream& operator<<(std::ostream& os, const DRegister& rhs); - -// Values for the condition field as defined in Table A8-1 "Condition -// codes" (refer to Section A8.3 "Conditional execution"). -enum Condition { // private marker to avoid generate-operator-out.py from processing. - kNoCondition = -1, - // Meaning (integer) | Meaning (floating-point) - // ---------------------------------------+----------------------------------------- - EQ = 0, // Equal | Equal - NE = 1, // Not equal | Not equal, or unordered - CS = 2, // Carry set | Greater than, equal, or unordered - CC = 3, // Carry clear | Less than - MI = 4, // Minus, negative | Less than - PL = 5, // Plus, positive or zero | Greater than, equal, or unordered - VS = 6, // Overflow | Unordered (i.e. at least one NaN operand) - VC = 7, // No overflow | Not unordered - HI = 8, // Unsigned higher | Greater than, or unordered - LS = 9, // Unsigned lower or same | Less than or equal - GE = 10, // Signed greater than or equal | Greater than or equal - LT = 11, // Signed less than | Less than, or unordered - GT = 12, // Signed greater than | Greater than - LE = 13, // Signed less than or equal | Less than, equal, or unordered - AL = 14, // Always (unconditional) | Always (unconditional) - kSpecialCondition = 15, // Special condition (refer to Section A8.3 "Conditional execution"). - kMaxCondition = 16, - - HS = CS, // HS (unsigned higher or same) is a synonym for CS. - LO = CC // LO (unsigned lower) is a synonym for CC. -}; -std::ostream& operator<<(std::ostream& os, const Condition& rhs); - - // Opcodes for Data-processing instructions (instructions with a type 0 and 1) // as defined in section A3.4 enum Opcode { @@ -151,70 +120,6 @@ enum Opcode { ORN = 16, // Logical OR NOT. kMaxOperand = 17 }; -std::ostream& operator<<(std::ostream& os, const Opcode& rhs); - -// Shifter types for Data-processing operands as defined in section A5.1.2. -enum Shift { - kNoShift = -1, - LSL = 0, // Logical shift left - LSR = 1, // Logical shift right - ASR = 2, // Arithmetic shift right - ROR = 3, // Rotate right - RRX = 4, // Rotate right with extend. - kMaxShift -}; -std::ostream& operator<<(std::ostream& os, const Shift& rhs); - -// Constants used for the decoding or encoding of the individual fields of -// instructions. Based on the "Figure 3-1 ARM instruction set summary". -enum InstructionFields { // private marker to avoid generate-operator-out.py from processing. - kConditionShift = 28, - kConditionBits = 4, - kTypeShift = 25, - kTypeBits = 3, - kLinkShift = 24, - kLinkBits = 1, - kUShift = 23, - kUBits = 1, - kOpcodeShift = 21, - kOpcodeBits = 4, - kSShift = 20, - kSBits = 1, - kRnShift = 16, - kRnBits = 4, - kRdShift = 12, - kRdBits = 4, - kRsShift = 8, - kRsBits = 4, - kRmShift = 0, - kRmBits = 4, - - // Immediate instruction fields encoding. - kRotateShift = 8, - kRotateBits = 4, - kImmed8Shift = 0, - kImmed8Bits = 8, - - // Shift instruction register fields encodings. - kShiftImmShift = 7, - kShiftRegisterShift = 8, - kShiftImmBits = 5, - kShiftShift = 5, - kShiftBits = 2, - - // Load/store instruction offset field encoding. - kOffset12Shift = 0, - kOffset12Bits = 12, - kOffset12Mask = 0x00000fff, - - // Mul instruction register fields encodings. - kMulRdShift = 16, - kMulRdBits = 4, - kMulRnShift = 12, - kMulRnBits = 4, - - kBranchOffsetMask = 0x00ffffff -}; // Size (in bytes) of registers. const int kRegisterSize = 4; @@ -222,231 +127,6 @@ const int kRegisterSize = 4; // List of registers used in load/store multiple. typedef uint16_t RegList; -// The class Instr enables access to individual fields defined in the ARM -// architecture instruction set encoding as described in figure A3-1. -// -// Example: Test whether the instruction at ptr does set the condition code -// bits. -// -// bool InstructionSetsConditionCodes(uint8_t* ptr) { -// Instr* instr = Instr::At(ptr); -// int type = instr->TypeField(); -// return ((type == 0) || (type == 1)) && instr->HasS(); -// } -// -class Instr { - public: - enum { - kInstrSize = 4, - kInstrSizeLog2 = 2, - kPCReadOffset = 8 - }; - - bool IsBreakPoint() { - return IsBkpt(); - } - - // Get the raw instruction bits. - int32_t InstructionBits() const { - return *reinterpret_cast<const int32_t*>(this); - } - - // Set the raw instruction bits to value. - void SetInstructionBits(int32_t value) { - *reinterpret_cast<int32_t*>(this) = value; - } - - // Read one particular bit out of the instruction bits. - int Bit(int nr) const { - return (InstructionBits() >> nr) & 1; - } - - // Read a bit field out of the instruction bits. - int Bits(int shift, int count) const { - return (InstructionBits() >> shift) & ((1 << count) - 1); - } - - - // Accessors for the different named fields used in the ARM encoding. - // The naming of these accessor corresponds to figure A3-1. - // Generally applicable fields - Condition ConditionField() const { - return static_cast<Condition>(Bits(kConditionShift, kConditionBits)); - } - int TypeField() const { return Bits(kTypeShift, kTypeBits); } - - Register RnField() const { return static_cast<Register>( - Bits(kRnShift, kRnBits)); } - Register RdField() const { return static_cast<Register>( - Bits(kRdShift, kRdBits)); } - - // Fields used in Data processing instructions - Opcode OpcodeField() const { - return static_cast<Opcode>(Bits(kOpcodeShift, kOpcodeBits)); - } - int SField() const { return Bits(kSShift, kSBits); } - // with register - Register RmField() const { - return static_cast<Register>(Bits(kRmShift, kRmBits)); - } - Shift ShiftField() const { return static_cast<Shift>( - Bits(kShiftShift, kShiftBits)); } - int RegShiftField() const { return Bit(4); } - Register RsField() const { - return static_cast<Register>(Bits(kRsShift, kRsBits)); - } - int ShiftAmountField() const { return Bits(kShiftImmShift, - kShiftImmBits); } - // with immediate - int RotateField() const { return Bits(kRotateShift, kRotateBits); } - int Immed8Field() const { return Bits(kImmed8Shift, kImmed8Bits); } - - // Fields used in Load/Store instructions - int PUField() const { return Bits(23, 2); } - int BField() const { return Bit(22); } - int WField() const { return Bit(21); } - int LField() const { return Bit(20); } - // with register uses same fields as Data processing instructions above - // with immediate - int Offset12Field() const { return Bits(kOffset12Shift, - kOffset12Bits); } - // multiple - int RlistField() const { return Bits(0, 16); } - // extra loads and stores - int SignField() const { return Bit(6); } - int HField() const { return Bit(5); } - int ImmedHField() const { return Bits(8, 4); } - int ImmedLField() const { return Bits(0, 4); } - - // Fields used in Branch instructions - int LinkField() const { return Bits(kLinkShift, kLinkBits); } - int SImmed24Field() const { return ((InstructionBits() << 8) >> 8); } - - // Fields used in Supervisor Call instructions - uint32_t SvcField() const { return Bits(0, 24); } - - // Field used in Breakpoint instruction - uint16_t BkptField() const { - return ((Bits(8, 12) << 4) | Bits(0, 4)); - } - - // Field used in 16-bit immediate move instructions - uint16_t MovwField() const { - return ((Bits(16, 4) << 12) | Bits(0, 12)); - } - - // Field used in VFP float immediate move instruction - float ImmFloatField() const { - uint32_t imm32 = (Bit(19) << 31) | (((1 << 5) - Bit(18)) << 25) | - (Bits(16, 2) << 23) | (Bits(0, 4) << 19); - return bit_cast<float, uint32_t>(imm32); - } - - // Field used in VFP double immediate move instruction - double ImmDoubleField() const { - uint64_t imm64 = (Bit(19)*(1LL << 63)) | (((1LL << 8) - Bit(18)) << 54) | - (Bits(16, 2)*(1LL << 52)) | (Bits(0, 4)*(1LL << 48)); - return bit_cast<double, uint64_t>(imm64); - } - - // Test for data processing instructions of type 0 or 1. - // See "ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition", - // section A5.1 "ARM instruction set encoding". - bool IsDataProcessing() const { - CHECK_NE(ConditionField(), kSpecialCondition); - CHECK_EQ(Bits(26, 2), 0); // Type 0 or 1. - return ((Bits(20, 5) & 0x19) != 0x10) && - ((Bit(25) == 1) || // Data processing immediate. - (Bit(4) == 0) || // Data processing register. - (Bit(7) == 0)); // Data processing register-shifted register. - } - - // Tests for special encodings of type 0 instructions (extra loads and stores, - // as well as multiplications, synchronization primitives, and miscellaneous). - // Can only be called for a type 0 or 1 instruction. - bool IsMiscellaneous() const { - CHECK_EQ(Bits(26, 2), 0); // Type 0 or 1. - return ((Bit(25) == 0) && ((Bits(20, 5) & 0x19) == 0x10) && (Bit(7) == 0)); - } - bool IsMultiplyOrSyncPrimitive() const { - CHECK_EQ(Bits(26, 2), 0); // Type 0 or 1. - return ((Bit(25) == 0) && (Bits(4, 4) == 9)); - } - - // Test for Supervisor Call instruction. - bool IsSvc() const { - return ((InstructionBits() & 0xff000000) == 0xef000000); - } - - // Test for Breakpoint instruction. - bool IsBkpt() const { - return ((InstructionBits() & 0xfff000f0) == 0xe1200070); - } - - // VFP register fields. - SRegister SnField() const { - return static_cast<SRegister>((Bits(kRnShift, kRnBits) << 1) + Bit(7)); - } - SRegister SdField() const { - return static_cast<SRegister>((Bits(kRdShift, kRdBits) << 1) + Bit(22)); - } - SRegister SmField() const { - return static_cast<SRegister>((Bits(kRmShift, kRmBits) << 1) + Bit(5)); - } - DRegister DnField() const { - return static_cast<DRegister>(Bits(kRnShift, kRnBits) + (Bit(7) << 4)); - } - DRegister DdField() const { - return static_cast<DRegister>(Bits(kRdShift, kRdBits) + (Bit(22) << 4)); - } - DRegister DmField() const { - return static_cast<DRegister>(Bits(kRmShift, kRmBits) + (Bit(5) << 4)); - } - - // Test for VFP data processing or single transfer instructions of type 7. - bool IsVFPDataProcessingOrSingleTransfer() const { - CHECK_NE(ConditionField(), kSpecialCondition); - CHECK_EQ(TypeField(), 7); - return ((Bit(24) == 0) && (Bits(9, 3) == 5)); - // Bit(4) == 0: Data Processing - // Bit(4) == 1: 8, 16, or 32-bit Transfer between ARM Core and VFP - } - - // Test for VFP 64-bit transfer instructions of type 6. - bool IsVFPDoubleTransfer() const { - CHECK_NE(ConditionField(), kSpecialCondition); - CHECK_EQ(TypeField(), 6); - return ((Bits(21, 4) == 2) && (Bits(9, 3) == 5) && - ((Bits(4, 4) & 0xd) == 1)); - } - - // Test for VFP load and store instructions of type 6. - bool IsVFPLoadStore() const { - CHECK_NE(ConditionField(), kSpecialCondition); - CHECK_EQ(TypeField(), 6); - return ((Bits(20, 5) & 0x12) == 0x10) && (Bits(9, 3) == 5); - } - - // Special accessors that test for existence of a value. - bool HasS() const { return SField() == 1; } - bool HasB() const { return BField() == 1; } - bool HasW() const { return WField() == 1; } - bool HasL() const { return LField() == 1; } - bool HasSign() const { return SignField() == 1; } - bool HasH() const { return HField() == 1; } - bool HasLink() const { return LinkField() == 1; } - - // Instructions are read out of a code stream. The only way to get a - // reference to an instruction is to convert a pointer. There is no way - // to allocate or create instances of class Instr. - // Use the At(pc) function to create references to Instr. - static Instr* At(uintptr_t pc) { return reinterpret_cast<Instr*>(pc); } - Instr* Next() { return this + kInstrSize; } - - private: - // We need to prevent the creation of instances of class Instr. - DISALLOW_IMPLICIT_CONSTRUCTORS(Instr); -}; } // namespace arm } // namespace art diff --git a/compiler/utils/arm/jni_macro_assembler_arm.cc b/compiler/utils/arm/jni_macro_assembler_arm.cc deleted file mode 100644 index 3f425dfaf5..0000000000 --- a/compiler/utils/arm/jni_macro_assembler_arm.cc +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "jni_macro_assembler_arm.h" - -#include <algorithm> - -#include "assembler_thumb2.h" -#include "base/arena_allocator.h" -#include "base/bit_utils.h" -#include "base/logging.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "offsets.h" -#include "thread.h" - -namespace art { -namespace arm { - -constexpr size_t kFramePointerSize = static_cast<size_t>(kArmPointerSize); - -// Slowpath entered when Thread::Current()->_exception is non-null -class ArmExceptionSlowPath FINAL : public SlowPath { - public: - ArmExceptionSlowPath(ArmManagedRegister scratch, size_t stack_adjust) - : scratch_(scratch), stack_adjust_(stack_adjust) { - } - void Emit(Assembler *sp_asm) OVERRIDE; - private: - const ArmManagedRegister scratch_; - const size_t stack_adjust_; -}; - -ArmJNIMacroAssembler::ArmJNIMacroAssembler(ArenaAllocator* arena, InstructionSet isa) { - switch (isa) { - case kArm: - case kThumb2: - asm_.reset(new (arena) Thumb2Assembler(arena)); - break; - - default: - LOG(FATAL) << isa; - UNREACHABLE(); - } -} - -ArmJNIMacroAssembler::~ArmJNIMacroAssembler() { -} - -size_t ArmJNIMacroAssembler::CodeSize() const { - return asm_->CodeSize(); -} - -DebugFrameOpCodeWriterForAssembler& ArmJNIMacroAssembler::cfi() { - return asm_->cfi(); -} - -void ArmJNIMacroAssembler::FinalizeCode() { - asm_->FinalizeCode(); -} - -void ArmJNIMacroAssembler::FinalizeInstructions(const MemoryRegion& region) { - asm_->FinalizeInstructions(region); -} - -static dwarf::Reg DWARFReg(Register reg) { - return dwarf::Reg::ArmCore(static_cast<int>(reg)); -} - -static dwarf::Reg DWARFReg(SRegister reg) { - return dwarf::Reg::ArmFp(static_cast<int>(reg)); -} - -#define __ asm_-> - -void ArmJNIMacroAssembler::BuildFrame(size_t frame_size, - ManagedRegister method_reg, - ArrayRef<const ManagedRegister> callee_save_regs, - const ManagedRegisterEntrySpills& entry_spills) { - CHECK_EQ(CodeSize(), 0U); // Nothing emitted yet - CHECK_ALIGNED(frame_size, kStackAlignment); - CHECK_EQ(R0, method_reg.AsArm().AsCoreRegister()); - - // Push callee saves and link register. - RegList core_spill_mask = 1 << LR; - uint32_t fp_spill_mask = 0; - for (const ManagedRegister& reg : callee_save_regs) { - if (reg.AsArm().IsCoreRegister()) { - core_spill_mask |= 1 << reg.AsArm().AsCoreRegister(); - } else { - fp_spill_mask |= 1 << reg.AsArm().AsSRegister(); - } - } - __ PushList(core_spill_mask); - cfi().AdjustCFAOffset(POPCOUNT(core_spill_mask) * kFramePointerSize); - cfi().RelOffsetForMany(DWARFReg(Register(0)), 0, core_spill_mask, kFramePointerSize); - if (fp_spill_mask != 0) { - __ vpushs(SRegister(CTZ(fp_spill_mask)), POPCOUNT(fp_spill_mask)); - cfi().AdjustCFAOffset(POPCOUNT(fp_spill_mask) * kFramePointerSize); - cfi().RelOffsetForMany(DWARFReg(SRegister(0)), 0, fp_spill_mask, kFramePointerSize); - } - - // Increase frame to required size. - int pushed_values = POPCOUNT(core_spill_mask) + POPCOUNT(fp_spill_mask); - CHECK_GT(frame_size, pushed_values * kFramePointerSize); // Must at least have space for Method*. - IncreaseFrameSize(frame_size - pushed_values * kFramePointerSize); // handles CFI as well. - - // Write out Method*. - __ StoreToOffset(kStoreWord, R0, SP, 0); - - // Write out entry spills. - int32_t offset = frame_size + kFramePointerSize; - for (size_t i = 0; i < entry_spills.size(); ++i) { - ArmManagedRegister reg = entry_spills.at(i).AsArm(); - if (reg.IsNoRegister()) { - // only increment stack offset. - ManagedRegisterSpill spill = entry_spills.at(i); - offset += spill.getSize(); - } else if (reg.IsCoreRegister()) { - __ StoreToOffset(kStoreWord, reg.AsCoreRegister(), SP, offset); - offset += 4; - } else if (reg.IsSRegister()) { - __ StoreSToOffset(reg.AsSRegister(), SP, offset); - offset += 4; - } else if (reg.IsDRegister()) { - __ StoreDToOffset(reg.AsDRegister(), SP, offset); - offset += 8; - } - } -} - -void ArmJNIMacroAssembler::RemoveFrame(size_t frame_size, - ArrayRef<const ManagedRegister> callee_save_regs) { - CHECK_ALIGNED(frame_size, kStackAlignment); - cfi().RememberState(); - - // Compute callee saves to pop and PC. - RegList core_spill_mask = 1 << PC; - uint32_t fp_spill_mask = 0; - for (const ManagedRegister& reg : callee_save_regs) { - if (reg.AsArm().IsCoreRegister()) { - core_spill_mask |= 1 << reg.AsArm().AsCoreRegister(); - } else { - fp_spill_mask |= 1 << reg.AsArm().AsSRegister(); - } - } - - // Decrease frame to start of callee saves. - int pop_values = POPCOUNT(core_spill_mask) + POPCOUNT(fp_spill_mask); - CHECK_GT(frame_size, pop_values * kFramePointerSize); - DecreaseFrameSize(frame_size - (pop_values * kFramePointerSize)); // handles CFI as well. - - if (fp_spill_mask != 0) { - __ vpops(SRegister(CTZ(fp_spill_mask)), POPCOUNT(fp_spill_mask)); - cfi().AdjustCFAOffset(-kFramePointerSize * POPCOUNT(fp_spill_mask)); - cfi().RestoreMany(DWARFReg(SRegister(0)), fp_spill_mask); - } - - // Pop callee saves and PC. - __ PopList(core_spill_mask); - - // The CFI should be restored for any code that follows the exit block. - cfi().RestoreState(); - cfi().DefCFAOffset(frame_size); -} - -void ArmJNIMacroAssembler::IncreaseFrameSize(size_t adjust) { - __ AddConstant(SP, -adjust); - cfi().AdjustCFAOffset(adjust); -} - -static void DecreaseFrameSizeImpl(ArmAssembler* assembler, size_t adjust) { - assembler->AddConstant(SP, adjust); - assembler->cfi().AdjustCFAOffset(-adjust); -} - -void ArmJNIMacroAssembler::DecreaseFrameSize(size_t adjust) { - DecreaseFrameSizeImpl(asm_.get(), adjust); -} - -void ArmJNIMacroAssembler::Store(FrameOffset dest, ManagedRegister msrc, size_t size) { - ArmManagedRegister src = msrc.AsArm(); - if (src.IsNoRegister()) { - CHECK_EQ(0u, size); - } else if (src.IsCoreRegister()) { - CHECK_EQ(4u, size); - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); - } else if (src.IsRegisterPair()) { - CHECK_EQ(8u, size); - __ StoreToOffset(kStoreWord, src.AsRegisterPairLow(), SP, dest.Int32Value()); - __ StoreToOffset(kStoreWord, src.AsRegisterPairHigh(), SP, dest.Int32Value() + 4); - } else if (src.IsSRegister()) { - __ StoreSToOffset(src.AsSRegister(), SP, dest.Int32Value()); - } else { - CHECK(src.IsDRegister()) << src; - __ StoreDToOffset(src.AsDRegister(), SP, dest.Int32Value()); - } -} - -void ArmJNIMacroAssembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { - ArmManagedRegister src = msrc.AsArm(); - CHECK(src.IsCoreRegister()) << src; - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { - ArmManagedRegister src = msrc.AsArm(); - CHECK(src.IsCoreRegister()) << src; - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreSpanning(FrameOffset dest, - ManagedRegister msrc, - FrameOffset in_off, - ManagedRegister mscratch) { - ArmManagedRegister src = msrc.AsArm(); - ArmManagedRegister scratch = mscratch.AsArm(); - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, in_off.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value() + sizeof(uint32_t)); -} - -void ArmJNIMacroAssembler::CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::LoadRef(ManagedRegister mdest, - ManagedRegister mbase, - MemberOffset offs, - bool unpoison_reference) { - ArmManagedRegister base = mbase.AsArm(); - ArmManagedRegister dst = mdest.AsArm(); - CHECK(base.IsCoreRegister()) << base; - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, - dst.AsCoreRegister(), - base.AsCoreRegister(), - offs.Int32Value()); - if (unpoison_reference) { - __ MaybeUnpoisonHeapReference(dst.AsCoreRegister()); - } -} - -void ArmJNIMacroAssembler::LoadRef(ManagedRegister mdest, FrameOffset src) { - ArmManagedRegister dst = mdest.AsArm(); - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, dst.AsCoreRegister(), SP, src.Int32Value()); -} - -void ArmJNIMacroAssembler::LoadRawPtr(ManagedRegister mdest, - ManagedRegister mbase, - Offset offs) { - ArmManagedRegister base = mbase.AsArm(); - ArmManagedRegister dst = mdest.AsArm(); - CHECK(base.IsCoreRegister()) << base; - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, - dst.AsCoreRegister(), - base.AsCoreRegister(), - offs.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreImmediateToFrame(FrameOffset dest, - uint32_t imm, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadImmediate(scratch.AsCoreRegister(), imm); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); -} - -static void EmitLoad(ArmAssembler* assembler, - ManagedRegister m_dst, - Register src_register, - int32_t src_offset, - size_t size) { - ArmManagedRegister dst = m_dst.AsArm(); - if (dst.IsNoRegister()) { - CHECK_EQ(0u, size) << dst; - } else if (dst.IsCoreRegister()) { - CHECK_EQ(4u, size) << dst; - assembler->LoadFromOffset(kLoadWord, dst.AsCoreRegister(), src_register, src_offset); - } else if (dst.IsRegisterPair()) { - CHECK_EQ(8u, size) << dst; - assembler->LoadFromOffset(kLoadWord, dst.AsRegisterPairLow(), src_register, src_offset); - assembler->LoadFromOffset(kLoadWord, dst.AsRegisterPairHigh(), src_register, src_offset + 4); - } else if (dst.IsSRegister()) { - assembler->LoadSFromOffset(dst.AsSRegister(), src_register, src_offset); - } else { - CHECK(dst.IsDRegister()) << dst; - assembler->LoadDFromOffset(dst.AsDRegister(), src_register, src_offset); - } -} - -void ArmJNIMacroAssembler::Load(ManagedRegister m_dst, FrameOffset src, size_t size) { - EmitLoad(asm_.get(), m_dst, SP, src.Int32Value(), size); -} - -void ArmJNIMacroAssembler::LoadFromThread(ManagedRegister m_dst, ThreadOffset32 src, size_t size) { - EmitLoad(asm_.get(), m_dst, TR, src.Int32Value(), size); -} - -void ArmJNIMacroAssembler::LoadRawPtrFromThread(ManagedRegister m_dst, ThreadOffset32 offs) { - ArmManagedRegister dst = m_dst.AsArm(); - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, dst.AsCoreRegister(), TR, offs.Int32Value()); -} - -void ArmJNIMacroAssembler::CopyRawPtrFromThread(FrameOffset fr_offs, - ThreadOffset32 thr_offs, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), TR, thr_offs.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, fr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::CopyRawPtrToThread(ThreadOffset32 thr_offs, - FrameOffset fr_offs, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, fr_offs.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), TR, thr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreStackOffsetToThread(ThreadOffset32 thr_offs, - FrameOffset fr_offs, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ AddConstant(scratch.AsCoreRegister(), SP, fr_offs.Int32Value(), AL); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), TR, thr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreStackPointerToThread(ThreadOffset32 thr_offs) { - __ StoreToOffset(kStoreWord, SP, TR, thr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::SignExtend(ManagedRegister /*mreg*/, size_t /*size*/) { - UNIMPLEMENTED(FATAL) << "no sign extension necessary for arm"; -} - -void ArmJNIMacroAssembler::ZeroExtend(ManagedRegister /*mreg*/, size_t /*size*/) { - UNIMPLEMENTED(FATAL) << "no zero extension necessary for arm"; -} - -void ArmJNIMacroAssembler::Move(ManagedRegister m_dst, ManagedRegister m_src, size_t /*size*/) { - ArmManagedRegister dst = m_dst.AsArm(); - ArmManagedRegister src = m_src.AsArm(); - if (!dst.Equals(src)) { - if (dst.IsCoreRegister()) { - CHECK(src.IsCoreRegister()) << src; - __ mov(dst.AsCoreRegister(), ShifterOperand(src.AsCoreRegister())); - } else if (dst.IsDRegister()) { - if (src.IsDRegister()) { - __ vmovd(dst.AsDRegister(), src.AsDRegister()); - } else { - // VMOV Dn, Rlo, Rhi (Dn = {Rlo, Rhi}) - CHECK(src.IsRegisterPair()) << src; - __ vmovdrr(dst.AsDRegister(), src.AsRegisterPairLow(), src.AsRegisterPairHigh()); - } - } else if (dst.IsSRegister()) { - if (src.IsSRegister()) { - __ vmovs(dst.AsSRegister(), src.AsSRegister()); - } else { - // VMOV Sn, Rn (Sn = Rn) - CHECK(src.IsCoreRegister()) << src; - __ vmovsr(dst.AsSRegister(), src.AsCoreRegister()); - } - } else { - CHECK(dst.IsRegisterPair()) << dst; - CHECK(src.IsRegisterPair()) << src; - // Ensure that the first move doesn't clobber the input of the second. - if (src.AsRegisterPairHigh() != dst.AsRegisterPairLow()) { - __ mov(dst.AsRegisterPairLow(), ShifterOperand(src.AsRegisterPairLow())); - __ mov(dst.AsRegisterPairHigh(), ShifterOperand(src.AsRegisterPairHigh())); - } else { - __ mov(dst.AsRegisterPairHigh(), ShifterOperand(src.AsRegisterPairHigh())); - __ mov(dst.AsRegisterPairLow(), ShifterOperand(src.AsRegisterPairLow())); - } - } - } -} - -void ArmJNIMacroAssembler::Copy(FrameOffset dest, - FrameOffset src, - ManagedRegister mscratch, - size_t size) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - CHECK(size == 4 || size == 8) << size; - if (size == 4) { - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); - } else if (size == 8) { - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value() + 4); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value() + 4); - } -} - -void ArmJNIMacroAssembler::Copy(FrameOffset dest, - ManagedRegister src_base, - Offset src_offset, - ManagedRegister mscratch, - size_t size) { - Register scratch = mscratch.AsArm().AsCoreRegister(); - CHECK_EQ(size, 4u); - __ LoadFromOffset(kLoadWord, scratch, src_base.AsArm().AsCoreRegister(), src_offset.Int32Value()); - __ StoreToOffset(kStoreWord, scratch, SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::Copy(ManagedRegister dest_base, - Offset dest_offset, - FrameOffset src, - ManagedRegister mscratch, - size_t size) { - Register scratch = mscratch.AsArm().AsCoreRegister(); - CHECK_EQ(size, 4u); - __ LoadFromOffset(kLoadWord, scratch, SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, - scratch, - dest_base.AsArm().AsCoreRegister(), - dest_offset.Int32Value()); -} - -void ArmJNIMacroAssembler::Copy(FrameOffset /*dst*/, - FrameOffset /*src_base*/, - Offset /*src_offset*/, - ManagedRegister /*mscratch*/, - size_t /*size*/) { - UNIMPLEMENTED(FATAL); -} - -void ArmJNIMacroAssembler::Copy(ManagedRegister dest, - Offset dest_offset, - ManagedRegister src, - Offset src_offset, - ManagedRegister mscratch, - size_t size) { - CHECK_EQ(size, 4u); - Register scratch = mscratch.AsArm().AsCoreRegister(); - __ LoadFromOffset(kLoadWord, scratch, src.AsArm().AsCoreRegister(), src_offset.Int32Value()); - __ StoreToOffset(kStoreWord, scratch, dest.AsArm().AsCoreRegister(), dest_offset.Int32Value()); -} - -void ArmJNIMacroAssembler::Copy(FrameOffset /*dst*/, - Offset /*dest_offset*/, - FrameOffset /*src*/, - Offset /*src_offset*/, - ManagedRegister /*scratch*/, - size_t /*size*/) { - UNIMPLEMENTED(FATAL); -} - -void ArmJNIMacroAssembler::CreateHandleScopeEntry(ManagedRegister mout_reg, - FrameOffset handle_scope_offset, - ManagedRegister min_reg, - bool null_allowed) { - ArmManagedRegister out_reg = mout_reg.AsArm(); - ArmManagedRegister in_reg = min_reg.AsArm(); - CHECK(in_reg.IsNoRegister() || in_reg.IsCoreRegister()) << in_reg; - CHECK(out_reg.IsCoreRegister()) << out_reg; - if (null_allowed) { - // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is - // the address in the handle scope holding the reference. - // e.g. out_reg = (handle == 0) ? 0 : (SP+handle_offset) - if (in_reg.IsNoRegister()) { - __ LoadFromOffset(kLoadWord, out_reg.AsCoreRegister(), SP, handle_scope_offset.Int32Value()); - in_reg = out_reg; - } - __ cmp(in_reg.AsCoreRegister(), ShifterOperand(0)); - if (!out_reg.Equals(in_reg)) { - __ it(EQ, kItElse); - __ LoadImmediate(out_reg.AsCoreRegister(), 0, EQ); - } else { - __ it(NE); - } - __ AddConstant(out_reg.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), NE); - } else { - __ AddConstant(out_reg.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), AL); - } -} - -void ArmJNIMacroAssembler::CreateHandleScopeEntry(FrameOffset out_off, - FrameOffset handle_scope_offset, - ManagedRegister mscratch, - bool null_allowed) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - if (null_allowed) { - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value()); - // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is - // the address in the handle scope holding the reference. - // e.g. scratch = (scratch == 0) ? 0 : (SP+handle_scope_offset) - __ cmp(scratch.AsCoreRegister(), ShifterOperand(0)); - __ it(NE); - __ AddConstant(scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), NE); - } else { - __ AddConstant(scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), AL); - } - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, out_off.Int32Value()); -} - -void ArmJNIMacroAssembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, - ManagedRegister min_reg) { - ArmManagedRegister out_reg = mout_reg.AsArm(); - ArmManagedRegister in_reg = min_reg.AsArm(); - CHECK(out_reg.IsCoreRegister()) << out_reg; - CHECK(in_reg.IsCoreRegister()) << in_reg; - Label null_arg; - if (!out_reg.Equals(in_reg)) { - __ LoadImmediate(out_reg.AsCoreRegister(), 0, EQ); // TODO: why EQ? - } - __ cmp(in_reg.AsCoreRegister(), ShifterOperand(0)); - __ it(NE); - __ LoadFromOffset(kLoadWord, out_reg.AsCoreRegister(), in_reg.AsCoreRegister(), 0, NE); -} - -void ArmJNIMacroAssembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { - // TODO: not validating references. -} - -void ArmJNIMacroAssembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { - // TODO: not validating references. -} - -void ArmJNIMacroAssembler::Call(ManagedRegister mbase, - Offset offset, - ManagedRegister mscratch) { - ArmManagedRegister base = mbase.AsArm(); - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(base.IsCoreRegister()) << base; - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadFromOffset(kLoadWord, - scratch.AsCoreRegister(), - base.AsCoreRegister(), - offset.Int32Value()); - __ blx(scratch.AsCoreRegister()); - // TODO: place reference map on call. -} - -void ArmJNIMacroAssembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - // Call *(*(SP + base) + offset) - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, base.Int32Value()); - __ LoadFromOffset(kLoadWord, - scratch.AsCoreRegister(), - scratch.AsCoreRegister(), - offset.Int32Value()); - __ blx(scratch.AsCoreRegister()); - // TODO: place reference map on call -} - -void ArmJNIMacroAssembler::CallFromThread(ThreadOffset32 offset ATTRIBUTE_UNUSED, - ManagedRegister scratch ATTRIBUTE_UNUSED) { - UNIMPLEMENTED(FATAL); -} - -void ArmJNIMacroAssembler::GetCurrentThread(ManagedRegister tr) { - __ mov(tr.AsArm().AsCoreRegister(), ShifterOperand(TR)); -} - -void ArmJNIMacroAssembler::GetCurrentThread(FrameOffset offset, ManagedRegister /*scratch*/) { - __ StoreToOffset(kStoreWord, TR, SP, offset.Int32Value(), AL); -} - -void ArmJNIMacroAssembler::ExceptionPoll(ManagedRegister mscratch, size_t stack_adjust) { - ArmManagedRegister scratch = mscratch.AsArm(); - ArmExceptionSlowPath* slow = new (__ GetArena()) ArmExceptionSlowPath(scratch, stack_adjust); - __ GetBuffer()->EnqueueSlowPath(slow); - __ LoadFromOffset(kLoadWord, - scratch.AsCoreRegister(), - TR, - Thread::ExceptionOffset<kArmPointerSize>().Int32Value()); - __ cmp(scratch.AsCoreRegister(), ShifterOperand(0)); - __ b(slow->Entry(), NE); -} - -std::unique_ptr<JNIMacroLabel> ArmJNIMacroAssembler::CreateLabel() { - return std::unique_ptr<JNIMacroLabel>(new ArmJNIMacroLabel()); -} - -void ArmJNIMacroAssembler::Jump(JNIMacroLabel* label) { - CHECK(label != nullptr); - __ b(ArmJNIMacroLabel::Cast(label)->AsArm()); -} - -void ArmJNIMacroAssembler::Jump(JNIMacroLabel* label, - JNIMacroUnaryCondition condition, - ManagedRegister test) { - CHECK(label != nullptr); - - arm::Condition arm_cond; - switch (condition) { - case JNIMacroUnaryCondition::kZero: - arm_cond = EQ; - break; - case JNIMacroUnaryCondition::kNotZero: - arm_cond = NE; - break; - default: - LOG(FATAL) << "Not implemented condition: " << static_cast<int>(condition); - UNREACHABLE(); - } - __ cmp(test.AsArm().AsCoreRegister(), ShifterOperand(0)); - __ b(ArmJNIMacroLabel::Cast(label)->AsArm(), arm_cond); -} - -void ArmJNIMacroAssembler::Bind(JNIMacroLabel* label) { - CHECK(label != nullptr); - __ Bind(ArmJNIMacroLabel::Cast(label)->AsArm()); -} - -#undef __ - -void ArmExceptionSlowPath::Emit(Assembler* sasm) { - ArmAssembler* sp_asm = down_cast<ArmAssembler*>(sasm); -#define __ sp_asm-> - __ Bind(&entry_); - if (stack_adjust_ != 0) { // Fix up the frame. - DecreaseFrameSizeImpl(sp_asm, stack_adjust_); - } - // Pass exception object as argument. - // Don't care about preserving R0 as this call won't return. - __ mov(R0, ShifterOperand(scratch_.AsCoreRegister())); - // Set up call to Thread::Current()->pDeliverException. - __ LoadFromOffset(kLoadWord, - R12, - TR, - QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, pDeliverException).Int32Value()); - __ blx(R12); -#undef __ -} - -void ArmJNIMacroAssembler::MemoryBarrier(ManagedRegister mscratch) { - CHECK_EQ(mscratch.AsArm().AsCoreRegister(), R12); - asm_->dmb(SY); -} - -} // namespace arm -} // namespace art diff --git a/compiler/utils/arm/jni_macro_assembler_arm.h b/compiler/utils/arm/jni_macro_assembler_arm.h deleted file mode 100644 index 809ac8be94..0000000000 --- a/compiler/utils/arm/jni_macro_assembler_arm.h +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_UTILS_ARM_JNI_MACRO_ASSEMBLER_ARM_H_ -#define ART_COMPILER_UTILS_ARM_JNI_MACRO_ASSEMBLER_ARM_H_ - -#include <memory> -#include <type_traits> -#include <vector> - -#include "arch/instruction_set.h" -#include "base/enums.h" -#include "base/macros.h" -#include "utils/jni_macro_assembler.h" -#include "utils/label.h" -#include "offsets.h" - -namespace art { -namespace arm { - -class ArmAssembler; - -class ArmJNIMacroAssembler : public JNIMacroAssembler<PointerSize::k32> { - public: - ArmJNIMacroAssembler(ArenaAllocator* arena, InstructionSet isa); - virtual ~ArmJNIMacroAssembler(); - - size_t CodeSize() const OVERRIDE; - DebugFrameOpCodeWriterForAssembler& cfi() OVERRIDE; - void FinalizeCode() OVERRIDE; - void FinalizeInstructions(const MemoryRegion& region) OVERRIDE; - - // - // Overridden common assembler high-level functionality - // - - // Emit code that will create an activation on the stack - void BuildFrame(size_t frame_size, - ManagedRegister method_reg, - ArrayRef<const ManagedRegister> callee_save_regs, - const ManagedRegisterEntrySpills& entry_spills) OVERRIDE; - - // Emit code that will remove an activation from the stack - void RemoveFrame(size_t frame_size, ArrayRef<const ManagedRegister> callee_save_regs) - OVERRIDE; - - void IncreaseFrameSize(size_t adjust) OVERRIDE; - void DecreaseFrameSize(size_t adjust) OVERRIDE; - - // Store routines - void Store(FrameOffset offs, ManagedRegister src, size_t size) OVERRIDE; - void StoreRef(FrameOffset dest, ManagedRegister src) OVERRIDE; - void StoreRawPtr(FrameOffset dest, ManagedRegister src) OVERRIDE; - - void StoreImmediateToFrame(FrameOffset dest, uint32_t imm, ManagedRegister scratch) OVERRIDE; - - void StoreStackOffsetToThread(ThreadOffset32 thr_offs, - FrameOffset fr_offs, - ManagedRegister scratch) OVERRIDE; - - void StoreStackPointerToThread(ThreadOffset32 thr_offs) OVERRIDE; - - void StoreSpanning(FrameOffset dest, ManagedRegister src, FrameOffset in_off, - ManagedRegister scratch) OVERRIDE; - - // Load routines - void Load(ManagedRegister dest, FrameOffset src, size_t size) OVERRIDE; - - void LoadFromThread(ManagedRegister dest, ThreadOffset32 src, size_t size) OVERRIDE; - - void LoadRef(ManagedRegister dest, FrameOffset src) OVERRIDE; - - void LoadRef(ManagedRegister dest, ManagedRegister base, MemberOffset offs, - bool unpoison_reference) OVERRIDE; - - void LoadRawPtr(ManagedRegister dest, ManagedRegister base, Offset offs) OVERRIDE; - - void LoadRawPtrFromThread(ManagedRegister dest, ThreadOffset32 offs) OVERRIDE; - - // Copying routines - void Move(ManagedRegister dest, ManagedRegister src, size_t size) OVERRIDE; - - void CopyRawPtrFromThread(FrameOffset fr_offs, - ThreadOffset32 thr_offs, - ManagedRegister scratch) OVERRIDE; - - void CopyRawPtrToThread(ThreadOffset32 thr_offs, FrameOffset fr_offs, ManagedRegister scratch) - OVERRIDE; - - void CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister scratch) OVERRIDE; - - void Copy(FrameOffset dest, FrameOffset src, ManagedRegister scratch, size_t size) OVERRIDE; - - void Copy(FrameOffset dest, ManagedRegister src_base, Offset src_offset, ManagedRegister scratch, - size_t size) OVERRIDE; - - void Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, ManagedRegister scratch, - size_t size) OVERRIDE; - - void Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, ManagedRegister scratch, - size_t size) OVERRIDE; - - void Copy(ManagedRegister dest, Offset dest_offset, ManagedRegister src, Offset src_offset, - ManagedRegister scratch, size_t size) OVERRIDE; - - void Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, - ManagedRegister scratch, size_t size) OVERRIDE; - - // Sign extension - void SignExtend(ManagedRegister mreg, size_t size) OVERRIDE; - - // Zero extension - void ZeroExtend(ManagedRegister mreg, size_t size) OVERRIDE; - - // Exploit fast access in managed code to Thread::Current() - void GetCurrentThread(ManagedRegister tr) OVERRIDE; - void GetCurrentThread(FrameOffset dest_offset, ManagedRegister scratch) OVERRIDE; - - // Set up out_reg to hold a Object** into the handle scope, or to be null if the - // value is null and null_allowed. in_reg holds a possibly stale reference - // that can be used to avoid loading the handle scope entry to see if the value is - // null. - void CreateHandleScopeEntry(ManagedRegister out_reg, FrameOffset handlescope_offset, - ManagedRegister in_reg, bool null_allowed) OVERRIDE; - - // Set up out_off to hold a Object** into the handle scope, or to be null if the - // value is null and null_allowed. - void CreateHandleScopeEntry(FrameOffset out_off, FrameOffset handlescope_offset, - ManagedRegister scratch, bool null_allowed) OVERRIDE; - - // src holds a handle scope entry (Object**) load this into dst - void LoadReferenceFromHandleScope(ManagedRegister dst, ManagedRegister src) OVERRIDE; - - // Heap::VerifyObject on src. In some cases (such as a reference to this) we - // know that src may not be null. - void VerifyObject(ManagedRegister src, bool could_be_null) OVERRIDE; - void VerifyObject(FrameOffset src, bool could_be_null) OVERRIDE; - - // Call to address held at [base+offset] - void Call(ManagedRegister base, Offset offset, ManagedRegister scratch) OVERRIDE; - void Call(FrameOffset base, Offset offset, ManagedRegister scratch) OVERRIDE; - void CallFromThread(ThreadOffset32 offset, ManagedRegister scratch) OVERRIDE; - - // Generate code to check if Thread::Current()->exception_ is non-null - // and branch to a ExceptionSlowPath if it is. - void ExceptionPoll(ManagedRegister scratch, size_t stack_adjust) OVERRIDE; - - void MemoryBarrier(ManagedRegister scratch) OVERRIDE; - - // Create a new label that can be used with Jump/Bind calls. - std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE; - // Emit an unconditional jump to the label. - void Jump(JNIMacroLabel* label) OVERRIDE; - // Emit a conditional jump to the label by applying a unary condition test to the register. - void Jump(JNIMacroLabel* label, JNIMacroUnaryCondition cond, ManagedRegister test) OVERRIDE; - // Code at this offset will serve as the target for the Jump call. - void Bind(JNIMacroLabel* label) OVERRIDE; - - private: - std::unique_ptr<ArmAssembler> asm_; -}; - -class ArmJNIMacroLabel FINAL : public JNIMacroLabelCommon<ArmJNIMacroLabel, art::Label, kArm> { - public: - art::Label* AsArm() { - return AsPlatformLabel(); - } -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_UTILS_ARM_JNI_MACRO_ASSEMBLER_ARM_H_ diff --git a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc index d07c047253..bebe64c2b9 100644 --- a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc +++ b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc @@ -120,8 +120,8 @@ void ArmVIXLJNIMacroAssembler::RemoveFrame(size_t frame_size, CHECK_ALIGNED(frame_size, kStackAlignment); cfi().RememberState(); - // Compute callee saves to pop and PC. - RegList core_spill_mask = 1 << PC; + // Compute callee saves to pop and LR. + RegList core_spill_mask = 1 << LR; uint32_t fp_spill_mask = 0; for (const ManagedRegister& reg : callee_save_regs) { if (reg.AsArm().IsCoreRegister()) { @@ -136,6 +136,7 @@ void ArmVIXLJNIMacroAssembler::RemoveFrame(size_t frame_size, CHECK_GT(frame_size, pop_values * kFramePointerSize); DecreaseFrameSize(frame_size - (pop_values * kFramePointerSize)); // handles CFI as well. + // Pop FP callee saves. if (fp_spill_mask != 0) { uint32_t first = CTZ(fp_spill_mask); // Check that list is contiguous. @@ -146,9 +147,18 @@ void ArmVIXLJNIMacroAssembler::RemoveFrame(size_t frame_size, cfi().RestoreMany(DWARFReg(s0), fp_spill_mask); } - // Pop callee saves and PC. + // Pop core callee saves and LR. ___ Pop(RegisterList(core_spill_mask)); + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Refresh Mark Register. + // TODO: Refresh MR only if suspend is taken. + ___ Ldr(mr, MemOperand(tr, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value())); + } + + // Return to LR. + ___ Bx(vixl32::lr); + // The CFI should be restored for any code that follows the exit block. cfi().RestoreState(); cfi().DefCFAOffset(frame_size); diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.cc b/compiler/utils/arm64/jni_macro_assembler_arm64.cc index 9cd6884cbe..bab84bea4c 100644 --- a/compiler/utils/arm64/jni_macro_assembler_arm64.cc +++ b/compiler/utils/arm64/jni_macro_assembler_arm64.cc @@ -772,10 +772,17 @@ void Arm64JNIMacroAssembler::RemoveFrame(size_t frame_size, asm_.UnspillRegisters(core_reg_list, frame_size - core_reg_size); asm_.UnspillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size); + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Refresh Mark Register. + // TODO: Refresh MR only if suspend is taken. + ___ Ldr(reg_w(MR), + MemOperand(reg_x(TR), Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value())); + } + // Decrease frame size to start of callee saved regs. DecreaseFrameSize(frame_size); - // Pop callee saved and return to LR. + // Return to LR. ___ Ret(); // The CFI should be restored for any code that follows the exit block. diff --git a/compiler/utils/assembler.cc b/compiler/utils/assembler.cc index 57f3b1570a..25eca23af6 100644 --- a/compiler/utils/assembler.cc +++ b/compiler/utils/assembler.cc @@ -19,24 +19,6 @@ #include <algorithm> #include <vector> -#ifdef ART_ENABLE_CODEGEN_arm -#include "arm/assembler_thumb2.h" -#endif -#ifdef ART_ENABLE_CODEGEN_arm64 -#include "arm64/assembler_arm64.h" -#endif -#ifdef ART_ENABLE_CODEGEN_mips -#include "mips/assembler_mips.h" -#endif -#ifdef ART_ENABLE_CODEGEN_mips64 -#include "mips64/assembler_mips64.h" -#endif -#ifdef ART_ENABLE_CODEGEN_x86 -#include "x86/assembler_x86.h" -#endif -#ifdef ART_ENABLE_CODEGEN_x86_64 -#include "x86_64/assembler_x86_64.h" -#endif #include "base/casts.h" #include "globals.h" #include "memory_region.h" diff --git a/compiler/utils/assembler_thumb_test.cc b/compiler/utils/assembler_thumb_test.cc index 4e9b619979..741beab1d5 100644 --- a/compiler/utils/assembler_thumb_test.cc +++ b/compiler/utils/assembler_thumb_test.cc @@ -22,7 +22,6 @@ #include <sys/types.h> #include "gtest/gtest.h" -#include "utils/arm/assembler_thumb2.h" #include "jni/quick/calling_convention.h" #include "utils/arm/jni_macro_assembler_arm_vixl.h" @@ -176,1451 +175,18 @@ void DumpAndCheck(std::vector<uint8_t>& code, const char* testname, const char* #endif // ART_TARGET_ANDROID } -#define __ assembler-> - -void EmitAndCheck(arm::Thumb2Assembler* assembler, const char* testname, - const char* const* results) { - __ FinalizeCode(); - size_t cs = __ CodeSize(); - std::vector<uint8_t> managed_code(cs); - MemoryRegion code(&managed_code[0], managed_code.size()); - __ FinalizeInstructions(code); - - DumpAndCheck(managed_code, testname, results); -} - -void EmitAndCheck(arm::Thumb2Assembler* assembler, const char* testname) { - InitResults(); - std::map<std::string, const char* const*>::iterator results = test_results.find(testname); - ASSERT_NE(results, test_results.end()); - - EmitAndCheck(assembler, testname, results->second); -} - -#undef __ - -class Thumb2AssemblerTest : public ::testing::Test { - public: - Thumb2AssemblerTest() : pool(), arena(&pool), assembler(&arena) { } - - ArenaPool pool; - ArenaAllocator arena; - arm::Thumb2Assembler assembler; -}; - -#define __ assembler. - -TEST_F(Thumb2AssemblerTest, SimpleMov) { - __ movs(R0, ShifterOperand(R1)); - __ mov(R0, ShifterOperand(R1)); - __ mov(R8, ShifterOperand(R9)); - - __ mov(R0, ShifterOperand(1)); - __ mov(R8, ShifterOperand(9)); - - EmitAndCheck(&assembler, "SimpleMov"); -} - -TEST_F(Thumb2AssemblerTest, SimpleMov32) { - __ Force32Bit(); - - __ mov(R0, ShifterOperand(R1)); - __ mov(R8, ShifterOperand(R9)); - - EmitAndCheck(&assembler, "SimpleMov32"); -} - -TEST_F(Thumb2AssemblerTest, SimpleMovAdd) { - __ mov(R0, ShifterOperand(R1)); - __ adds(R0, R1, ShifterOperand(R2)); - __ add(R0, R1, ShifterOperand(0)); - - EmitAndCheck(&assembler, "SimpleMovAdd"); -} - -TEST_F(Thumb2AssemblerTest, DataProcessingRegister) { - // 32 bit variants using low registers. - __ mvn(R0, ShifterOperand(R1), AL, kCcKeep); - __ add(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ sub(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ and_(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ orr(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ orn(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ eor(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ bic(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ adc(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ sbc(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ rsb(R0, R1, ShifterOperand(R2), AL, kCcKeep); - __ teq(R0, ShifterOperand(R1)); - - // 16 bit variants using low registers. - __ movs(R0, ShifterOperand(R1)); - __ mov(R0, ShifterOperand(R1), AL, kCcKeep); - __ mvns(R0, ShifterOperand(R1)); - __ add(R0, R0, ShifterOperand(R1), AL, kCcKeep); - __ adds(R0, R1, ShifterOperand(R2)); - __ subs(R0, R1, ShifterOperand(R2)); - __ adcs(R0, R0, ShifterOperand(R1)); - __ sbcs(R0, R0, ShifterOperand(R1)); - __ ands(R0, R0, ShifterOperand(R1)); - __ orrs(R0, R0, ShifterOperand(R1)); - __ eors(R0, R0, ShifterOperand(R1)); - __ bics(R0, R0, ShifterOperand(R1)); - __ tst(R0, ShifterOperand(R1)); - __ cmp(R0, ShifterOperand(R1)); - __ cmn(R0, ShifterOperand(R1)); - - // 16-bit variants using high registers. - __ mov(R1, ShifterOperand(R8), AL, kCcKeep); - __ mov(R9, ShifterOperand(R0), AL, kCcKeep); - __ mov(R8, ShifterOperand(R9), AL, kCcKeep); - __ add(R1, R1, ShifterOperand(R8), AL, kCcKeep); - __ add(R9, R9, ShifterOperand(R0), AL, kCcKeep); - __ add(R8, R8, ShifterOperand(R9), AL, kCcKeep); - __ cmp(R0, ShifterOperand(R9)); - __ cmp(R8, ShifterOperand(R1)); - __ cmp(R9, ShifterOperand(R8)); - - // The 16-bit RSBS Rd, Rn, #0, also known as NEGS Rd, Rn is specified using - // an immediate (0) but emitted without any, so we test it here. - __ rsbs(R0, R1, ShifterOperand(0)); - __ rsbs(R0, R0, ShifterOperand(0)); // Check Rd == Rn code path. - - // 32 bit variants using high registers that would be 16-bit if using low registers. - __ movs(R0, ShifterOperand(R8)); - __ mvns(R0, ShifterOperand(R8)); - __ add(R0, R1, ShifterOperand(R8), AL, kCcKeep); - __ adds(R0, R1, ShifterOperand(R8)); - __ subs(R0, R1, ShifterOperand(R8)); - __ adcs(R0, R0, ShifterOperand(R8)); - __ sbcs(R0, R0, ShifterOperand(R8)); - __ ands(R0, R0, ShifterOperand(R8)); - __ orrs(R0, R0, ShifterOperand(R8)); - __ eors(R0, R0, ShifterOperand(R8)); - __ bics(R0, R0, ShifterOperand(R8)); - __ tst(R0, ShifterOperand(R8)); - __ cmn(R0, ShifterOperand(R8)); - __ rsbs(R0, R8, ShifterOperand(0)); // Check that this is not emitted as 16-bit. - __ rsbs(R8, R8, ShifterOperand(0)); // Check that this is not emitted as 16-bit (Rd == Rn). - - // 32-bit variants of instructions that would be 16-bit outside IT block. - __ it(arm::EQ); - __ mvns(R0, ShifterOperand(R1), arm::EQ); - __ it(arm::EQ); - __ adds(R0, R1, ShifterOperand(R2), arm::EQ); - __ it(arm::EQ); - __ subs(R0, R1, ShifterOperand(R2), arm::EQ); - __ it(arm::EQ); - __ adcs(R0, R0, ShifterOperand(R1), arm::EQ); - __ it(arm::EQ); - __ sbcs(R0, R0, ShifterOperand(R1), arm::EQ); - __ it(arm::EQ); - __ ands(R0, R0, ShifterOperand(R1), arm::EQ); - __ it(arm::EQ); - __ orrs(R0, R0, ShifterOperand(R1), arm::EQ); - __ it(arm::EQ); - __ eors(R0, R0, ShifterOperand(R1), arm::EQ); - __ it(arm::EQ); - __ bics(R0, R0, ShifterOperand(R1), arm::EQ); - - // 16-bit variants of instructions that would be 32-bit outside IT block. - __ it(arm::EQ); - __ mvn(R0, ShifterOperand(R1), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ add(R0, R1, ShifterOperand(R2), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ sub(R0, R1, ShifterOperand(R2), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ adc(R0, R0, ShifterOperand(R1), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ sbc(R0, R0, ShifterOperand(R1), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ and_(R0, R0, ShifterOperand(R1), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ orr(R0, R0, ShifterOperand(R1), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ eor(R0, R0, ShifterOperand(R1), arm::EQ, kCcKeep); - __ it(arm::EQ); - __ bic(R0, R0, ShifterOperand(R1), arm::EQ, kCcKeep); - - // 16 bit variants selected for the default kCcDontCare. - __ mov(R0, ShifterOperand(R1)); - __ mvn(R0, ShifterOperand(R1)); - __ add(R0, R0, ShifterOperand(R1)); - __ add(R0, R1, ShifterOperand(R2)); - __ sub(R0, R1, ShifterOperand(R2)); - __ adc(R0, R0, ShifterOperand(R1)); - __ sbc(R0, R0, ShifterOperand(R1)); - __ and_(R0, R0, ShifterOperand(R1)); - __ orr(R0, R0, ShifterOperand(R1)); - __ eor(R0, R0, ShifterOperand(R1)); - __ bic(R0, R0, ShifterOperand(R1)); - __ mov(R1, ShifterOperand(R8)); - __ mov(R9, ShifterOperand(R0)); - __ mov(R8, ShifterOperand(R9)); - __ add(R1, R1, ShifterOperand(R8)); - __ add(R9, R9, ShifterOperand(R0)); - __ add(R8, R8, ShifterOperand(R9)); - __ rsb(R0, R1, ShifterOperand(0)); - __ rsb(R0, R0, ShifterOperand(0)); - - // And an arbitrary 32-bit instruction using IP. - __ add(R12, R1, ShifterOperand(R0), AL, kCcKeep); - - EmitAndCheck(&assembler, "DataProcessingRegister"); -} - -TEST_F(Thumb2AssemblerTest, DataProcessingImmediate) { - __ mov(R0, ShifterOperand(0x55)); - __ mvn(R0, ShifterOperand(0x55)); - __ add(R0, R1, ShifterOperand(0x55)); - __ sub(R0, R1, ShifterOperand(0x55)); - __ and_(R0, R1, ShifterOperand(0x55)); - __ orr(R0, R1, ShifterOperand(0x55)); - __ orn(R0, R1, ShifterOperand(0x55)); - __ eor(R0, R1, ShifterOperand(0x55)); - __ bic(R0, R1, ShifterOperand(0x55)); - __ adc(R0, R1, ShifterOperand(0x55)); - __ sbc(R0, R1, ShifterOperand(0x55)); - __ rsb(R0, R1, ShifterOperand(0x55)); - - __ tst(R0, ShifterOperand(0x55)); - __ teq(R0, ShifterOperand(0x55)); - __ cmp(R0, ShifterOperand(0x55)); - __ cmn(R0, ShifterOperand(0x55)); - - __ add(R0, R1, ShifterOperand(5)); - __ sub(R0, R1, ShifterOperand(5)); - - __ movs(R0, ShifterOperand(0x55)); - __ mvns(R0, ShifterOperand(0x55)); - - __ adds(R0, R1, ShifterOperand(5)); - __ subs(R0, R1, ShifterOperand(5)); - - EmitAndCheck(&assembler, "DataProcessingImmediate"); -} - -TEST_F(Thumb2AssemblerTest, DataProcessingModifiedImmediate) { - __ mov(R0, ShifterOperand(0x550055)); - __ mvn(R0, ShifterOperand(0x550055)); - __ add(R0, R1, ShifterOperand(0x550055)); - __ sub(R0, R1, ShifterOperand(0x550055)); - __ and_(R0, R1, ShifterOperand(0x550055)); - __ orr(R0, R1, ShifterOperand(0x550055)); - __ orn(R0, R1, ShifterOperand(0x550055)); - __ eor(R0, R1, ShifterOperand(0x550055)); - __ bic(R0, R1, ShifterOperand(0x550055)); - __ adc(R0, R1, ShifterOperand(0x550055)); - __ sbc(R0, R1, ShifterOperand(0x550055)); - __ rsb(R0, R1, ShifterOperand(0x550055)); - - __ tst(R0, ShifterOperand(0x550055)); - __ teq(R0, ShifterOperand(0x550055)); - __ cmp(R0, ShifterOperand(0x550055)); - __ cmn(R0, ShifterOperand(0x550055)); - - EmitAndCheck(&assembler, "DataProcessingModifiedImmediate"); -} - - -TEST_F(Thumb2AssemblerTest, DataProcessingModifiedImmediates) { - __ mov(R0, ShifterOperand(0x550055)); - __ mov(R0, ShifterOperand(0x55005500)); - __ mov(R0, ShifterOperand(0x55555555)); - __ mov(R0, ShifterOperand(0xd5000000)); // rotated to first position - __ mov(R0, ShifterOperand(0x6a000000)); // rotated to second position - __ mov(R0, ShifterOperand(0x350)); // rotated to 2nd last position - __ mov(R0, ShifterOperand(0x1a8)); // rotated to last position - - EmitAndCheck(&assembler, "DataProcessingModifiedImmediates"); -} - -TEST_F(Thumb2AssemblerTest, DataProcessingShiftedRegister) { - // 16-bit variants. - __ movs(R3, ShifterOperand(R4, LSL, 4)); - __ movs(R3, ShifterOperand(R4, LSR, 5)); - __ movs(R3, ShifterOperand(R4, ASR, 6)); - - // 32-bit ROR because ROR immediate doesn't have the same 16-bit version as other shifts. - __ movs(R3, ShifterOperand(R4, ROR, 7)); - - // 32-bit RRX because RRX has no 16-bit version. - __ movs(R3, ShifterOperand(R4, RRX)); - - // 32 bit variants (not setting condition codes). - __ mov(R3, ShifterOperand(R4, LSL, 4), AL, kCcKeep); - __ mov(R3, ShifterOperand(R4, LSR, 5), AL, kCcKeep); - __ mov(R3, ShifterOperand(R4, ASR, 6), AL, kCcKeep); - __ mov(R3, ShifterOperand(R4, ROR, 7), AL, kCcKeep); - __ mov(R3, ShifterOperand(R4, RRX), AL, kCcKeep); - - // 32 bit variants (high registers). - __ movs(R8, ShifterOperand(R4, LSL, 4)); - __ movs(R8, ShifterOperand(R4, LSR, 5)); - __ movs(R8, ShifterOperand(R4, ASR, 6)); - __ movs(R8, ShifterOperand(R4, ROR, 7)); - __ movs(R8, ShifterOperand(R4, RRX)); - - EmitAndCheck(&assembler, "DataProcessingShiftedRegister"); -} - -TEST_F(Thumb2AssemblerTest, ShiftImmediate) { - // Note: This test produces the same results as DataProcessingShiftedRegister - // but it does so using shift functions instead of mov(). - - // 16-bit variants. - __ Lsl(R3, R4, 4); - __ Lsr(R3, R4, 5); - __ Asr(R3, R4, 6); - - // 32-bit ROR because ROR immediate doesn't have the same 16-bit version as other shifts. - __ Ror(R3, R4, 7); - - // 32-bit RRX because RRX has no 16-bit version. - __ Rrx(R3, R4); - - // 32 bit variants (not setting condition codes). - __ Lsl(R3, R4, 4, AL, kCcKeep); - __ Lsr(R3, R4, 5, AL, kCcKeep); - __ Asr(R3, R4, 6, AL, kCcKeep); - __ Ror(R3, R4, 7, AL, kCcKeep); - __ Rrx(R3, R4, AL, kCcKeep); - - // 32 bit variants (high registers). - __ Lsls(R8, R4, 4); - __ Lsrs(R8, R4, 5); - __ Asrs(R8, R4, 6); - __ Rors(R8, R4, 7); - __ Rrxs(R8, R4); - - EmitAndCheck(&assembler, "ShiftImmediate"); -} - -TEST_F(Thumb2AssemblerTest, BasicLoad) { - __ ldr(R3, Address(R4, 24)); - __ ldrb(R3, Address(R4, 24)); - __ ldrh(R3, Address(R4, 24)); - __ ldrsb(R3, Address(R4, 24)); - __ ldrsh(R3, Address(R4, 24)); - - __ ldr(R3, Address(SP, 24)); - - // 32 bit variants - __ ldr(R8, Address(R4, 24)); - __ ldrb(R8, Address(R4, 24)); - __ ldrh(R8, Address(R4, 24)); - __ ldrsb(R8, Address(R4, 24)); - __ ldrsh(R8, Address(R4, 24)); - - EmitAndCheck(&assembler, "BasicLoad"); -} - - -TEST_F(Thumb2AssemblerTest, BasicStore) { - __ str(R3, Address(R4, 24)); - __ strb(R3, Address(R4, 24)); - __ strh(R3, Address(R4, 24)); - - __ str(R3, Address(SP, 24)); - - // 32 bit variants. - __ str(R8, Address(R4, 24)); - __ strb(R8, Address(R4, 24)); - __ strh(R8, Address(R4, 24)); - - EmitAndCheck(&assembler, "BasicStore"); -} - -TEST_F(Thumb2AssemblerTest, ComplexLoad) { - __ ldr(R3, Address(R4, 24, Address::Mode::Offset)); - __ ldr(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ ldr(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ ldr(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ ldr(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ ldr(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - __ ldrb(R3, Address(R4, 24, Address::Mode::Offset)); - __ ldrb(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ ldrb(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ ldrb(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ ldrb(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ ldrb(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - __ ldrh(R3, Address(R4, 24, Address::Mode::Offset)); - __ ldrh(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ ldrh(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ ldrh(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ ldrh(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ ldrh(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - __ ldrsb(R3, Address(R4, 24, Address::Mode::Offset)); - __ ldrsb(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ ldrsb(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ ldrsb(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ ldrsb(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ ldrsb(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - __ ldrsh(R3, Address(R4, 24, Address::Mode::Offset)); - __ ldrsh(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ ldrsh(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ ldrsh(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ ldrsh(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ ldrsh(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - EmitAndCheck(&assembler, "ComplexLoad"); -} - - -TEST_F(Thumb2AssemblerTest, ComplexStore) { - __ str(R3, Address(R4, 24, Address::Mode::Offset)); - __ str(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ str(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ str(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ str(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ str(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - __ strb(R3, Address(R4, 24, Address::Mode::Offset)); - __ strb(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ strb(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ strb(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ strb(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ strb(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - __ strh(R3, Address(R4, 24, Address::Mode::Offset)); - __ strh(R3, Address(R4, 24, Address::Mode::PreIndex)); - __ strh(R3, Address(R4, 24, Address::Mode::PostIndex)); - __ strh(R3, Address(R4, 24, Address::Mode::NegOffset)); - __ strh(R3, Address(R4, 24, Address::Mode::NegPreIndex)); - __ strh(R3, Address(R4, 24, Address::Mode::NegPostIndex)); - - EmitAndCheck(&assembler, "ComplexStore"); -} - -TEST_F(Thumb2AssemblerTest, NegativeLoadStore) { - __ ldr(R3, Address(R4, -24, Address::Mode::Offset)); - __ ldr(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ ldr(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ ldr(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ ldr(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ ldr(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - __ ldrb(R3, Address(R4, -24, Address::Mode::Offset)); - __ ldrb(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ ldrb(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ ldrb(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ ldrb(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ ldrb(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - __ ldrh(R3, Address(R4, -24, Address::Mode::Offset)); - __ ldrh(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ ldrh(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ ldrh(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ ldrh(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ ldrh(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - __ ldrsb(R3, Address(R4, -24, Address::Mode::Offset)); - __ ldrsb(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ ldrsb(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ ldrsb(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ ldrsb(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ ldrsb(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - __ ldrsh(R3, Address(R4, -24, Address::Mode::Offset)); - __ ldrsh(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ ldrsh(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ ldrsh(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ ldrsh(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ ldrsh(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - __ str(R3, Address(R4, -24, Address::Mode::Offset)); - __ str(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ str(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ str(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ str(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ str(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - __ strb(R3, Address(R4, -24, Address::Mode::Offset)); - __ strb(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ strb(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ strb(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ strb(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ strb(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - __ strh(R3, Address(R4, -24, Address::Mode::Offset)); - __ strh(R3, Address(R4, -24, Address::Mode::PreIndex)); - __ strh(R3, Address(R4, -24, Address::Mode::PostIndex)); - __ strh(R3, Address(R4, -24, Address::Mode::NegOffset)); - __ strh(R3, Address(R4, -24, Address::Mode::NegPreIndex)); - __ strh(R3, Address(R4, -24, Address::Mode::NegPostIndex)); - - EmitAndCheck(&assembler, "NegativeLoadStore"); -} - -TEST_F(Thumb2AssemblerTest, SimpleLoadStoreDual) { - __ strd(R2, Address(R0, 24, Address::Mode::Offset)); - __ ldrd(R2, Address(R0, 24, Address::Mode::Offset)); - - EmitAndCheck(&assembler, "SimpleLoadStoreDual"); -} - -TEST_F(Thumb2AssemblerTest, ComplexLoadStoreDual) { - __ strd(R2, Address(R0, 24, Address::Mode::Offset)); - __ strd(R2, Address(R0, 24, Address::Mode::PreIndex)); - __ strd(R2, Address(R0, 24, Address::Mode::PostIndex)); - __ strd(R2, Address(R0, 24, Address::Mode::NegOffset)); - __ strd(R2, Address(R0, 24, Address::Mode::NegPreIndex)); - __ strd(R2, Address(R0, 24, Address::Mode::NegPostIndex)); - - __ ldrd(R2, Address(R0, 24, Address::Mode::Offset)); - __ ldrd(R2, Address(R0, 24, Address::Mode::PreIndex)); - __ ldrd(R2, Address(R0, 24, Address::Mode::PostIndex)); - __ ldrd(R2, Address(R0, 24, Address::Mode::NegOffset)); - __ ldrd(R2, Address(R0, 24, Address::Mode::NegPreIndex)); - __ ldrd(R2, Address(R0, 24, Address::Mode::NegPostIndex)); - - EmitAndCheck(&assembler, "ComplexLoadStoreDual"); -} - -TEST_F(Thumb2AssemblerTest, NegativeLoadStoreDual) { - __ strd(R2, Address(R0, -24, Address::Mode::Offset)); - __ strd(R2, Address(R0, -24, Address::Mode::PreIndex)); - __ strd(R2, Address(R0, -24, Address::Mode::PostIndex)); - __ strd(R2, Address(R0, -24, Address::Mode::NegOffset)); - __ strd(R2, Address(R0, -24, Address::Mode::NegPreIndex)); - __ strd(R2, Address(R0, -24, Address::Mode::NegPostIndex)); - - __ ldrd(R2, Address(R0, -24, Address::Mode::Offset)); - __ ldrd(R2, Address(R0, -24, Address::Mode::PreIndex)); - __ ldrd(R2, Address(R0, -24, Address::Mode::PostIndex)); - __ ldrd(R2, Address(R0, -24, Address::Mode::NegOffset)); - __ ldrd(R2, Address(R0, -24, Address::Mode::NegPreIndex)); - __ ldrd(R2, Address(R0, -24, Address::Mode::NegPostIndex)); - - EmitAndCheck(&assembler, "NegativeLoadStoreDual"); -} - -TEST_F(Thumb2AssemblerTest, SimpleBranch) { - Label l1; - __ mov(R0, ShifterOperand(2)); - __ Bind(&l1); - __ mov(R1, ShifterOperand(1)); - __ b(&l1); - Label l2; - __ b(&l2); - __ mov(R1, ShifterOperand(2)); - __ Bind(&l2); - __ mov(R0, ShifterOperand(3)); - - Label l3; - __ mov(R0, ShifterOperand(2)); - __ Bind(&l3); - __ mov(R1, ShifterOperand(1)); - __ b(&l3, EQ); - - Label l4; - __ b(&l4, EQ); - __ mov(R1, ShifterOperand(2)); - __ Bind(&l4); - __ mov(R0, ShifterOperand(3)); - - // 2 linked labels. - Label l5; - __ b(&l5); - __ mov(R1, ShifterOperand(4)); - __ b(&l5); - __ mov(R1, ShifterOperand(5)); - __ Bind(&l5); - __ mov(R0, ShifterOperand(6)); - - EmitAndCheck(&assembler, "SimpleBranch"); -} - -TEST_F(Thumb2AssemblerTest, LongBranch) { - __ Force32Bit(); - // 32 bit branches. - Label l1; - __ mov(R0, ShifterOperand(2)); - __ Bind(&l1); - __ mov(R1, ShifterOperand(1)); - __ b(&l1); - - Label l2; - __ b(&l2); - __ mov(R1, ShifterOperand(2)); - __ Bind(&l2); - __ mov(R0, ShifterOperand(3)); - - Label l3; - __ mov(R0, ShifterOperand(2)); - __ Bind(&l3); - __ mov(R1, ShifterOperand(1)); - __ b(&l3, EQ); - - Label l4; - __ b(&l4, EQ); - __ mov(R1, ShifterOperand(2)); - __ Bind(&l4); - __ mov(R0, ShifterOperand(3)); - - // 2 linked labels. - Label l5; - __ b(&l5); - __ mov(R1, ShifterOperand(4)); - __ b(&l5); - __ mov(R1, ShifterOperand(5)); - __ Bind(&l5); - __ mov(R0, ShifterOperand(6)); - - EmitAndCheck(&assembler, "LongBranch"); -} - -TEST_F(Thumb2AssemblerTest, LoadMultiple) { - // 16 bit. - __ ldm(DB_W, R4, (1 << R0 | 1 << R3)); - - // 32 bit. - __ ldm(DB_W, R4, (1 << LR | 1 << R11)); - __ ldm(DB, R4, (1 << LR | 1 << R11)); - - // Single reg is converted to ldr - __ ldm(DB_W, R4, (1 << R5)); - - EmitAndCheck(&assembler, "LoadMultiple"); -} - -TEST_F(Thumb2AssemblerTest, StoreMultiple) { - // 16 bit. - __ stm(IA_W, R4, (1 << R0 | 1 << R3)); - - // 32 bit. - __ stm(IA_W, R4, (1 << LR | 1 << R11)); - __ stm(IA, R4, (1 << LR | 1 << R11)); - - // Single reg is converted to str - __ stm(IA_W, R4, (1 << R5)); - __ stm(IA, R4, (1 << R5)); - - EmitAndCheck(&assembler, "StoreMultiple"); -} - -TEST_F(Thumb2AssemblerTest, MovWMovT) { - // Always 32 bit. - __ movw(R4, 0); - __ movw(R4, 0x34); - __ movw(R9, 0x34); - __ movw(R3, 0x1234); - __ movw(R9, 0xffff); - - // Always 32 bit. - __ movt(R0, 0); - __ movt(R0, 0x1234); - __ movt(R1, 0xffff); - - EmitAndCheck(&assembler, "MovWMovT"); -} - -TEST_F(Thumb2AssemblerTest, SpecialAddSub) { - __ add(R2, SP, ShifterOperand(0x50)); // 16 bit. - __ add(SP, SP, ShifterOperand(0x50)); // 16 bit. - __ add(R8, SP, ShifterOperand(0x50)); // 32 bit. - - __ add(R2, SP, ShifterOperand(0xf00)); // 32 bit due to imm size. - __ add(SP, SP, ShifterOperand(0xf00)); // 32 bit due to imm size. - __ add(SP, SP, ShifterOperand(0xffc)); // 32 bit due to imm size; encoding T4. - - __ sub(SP, SP, ShifterOperand(0x50)); // 16 bit - __ sub(R0, SP, ShifterOperand(0x50)); // 32 bit - __ sub(R8, SP, ShifterOperand(0x50)); // 32 bit. - - __ sub(SP, SP, ShifterOperand(0xf00)); // 32 bit due to imm size - __ sub(SP, SP, ShifterOperand(0xffc)); // 32 bit due to imm size; encoding T4. - - EmitAndCheck(&assembler, "SpecialAddSub"); -} - -TEST_F(Thumb2AssemblerTest, LoadFromOffset) { - __ LoadFromOffset(kLoadWord, R2, R4, 12); - __ LoadFromOffset(kLoadWord, R2, R4, 0xfff); - __ LoadFromOffset(kLoadWord, R2, R4, 0x1000); - __ LoadFromOffset(kLoadWord, R2, R4, 0x1000a4); - __ LoadFromOffset(kLoadWord, R2, R4, 0x101000); - __ LoadFromOffset(kLoadWord, R4, R4, 0x101000); - __ LoadFromOffset(kLoadUnsignedHalfword, R2, R4, 12); - __ LoadFromOffset(kLoadUnsignedHalfword, R2, R4, 0xfff); - __ LoadFromOffset(kLoadUnsignedHalfword, R2, R4, 0x1000); - __ LoadFromOffset(kLoadUnsignedHalfword, R2, R4, 0x1000a4); - __ LoadFromOffset(kLoadUnsignedHalfword, R2, R4, 0x101000); - __ LoadFromOffset(kLoadUnsignedHalfword, R4, R4, 0x101000); - __ LoadFromOffset(kLoadWordPair, R2, R4, 12); - __ LoadFromOffset(kLoadWordPair, R2, R4, 0x3fc); - __ LoadFromOffset(kLoadWordPair, R2, R4, 0x400); - __ LoadFromOffset(kLoadWordPair, R2, R4, 0x400a4); - __ LoadFromOffset(kLoadWordPair, R2, R4, 0x40400); - __ LoadFromOffset(kLoadWordPair, R4, R4, 0x40400); - - __ LoadFromOffset(kLoadWord, R0, R12, 12); // 32-bit because of R12. - __ LoadFromOffset(kLoadWord, R2, R4, 0xa4 - 0x100000); - - __ LoadFromOffset(kLoadSignedByte, R2, R4, 12); - __ LoadFromOffset(kLoadUnsignedByte, R2, R4, 12); - __ LoadFromOffset(kLoadSignedHalfword, R2, R4, 12); - - EmitAndCheck(&assembler, "LoadFromOffset"); -} - -TEST_F(Thumb2AssemblerTest, StoreToOffset) { - __ StoreToOffset(kStoreWord, R2, R4, 12); - __ StoreToOffset(kStoreWord, R2, R4, 0xfff); - __ StoreToOffset(kStoreWord, R2, R4, 0x1000); - __ StoreToOffset(kStoreWord, R2, R4, 0x1000a4); - __ StoreToOffset(kStoreWord, R2, R4, 0x101000); - __ StoreToOffset(kStoreWord, R4, R4, 0x101000); - __ StoreToOffset(kStoreHalfword, R2, R4, 12); - __ StoreToOffset(kStoreHalfword, R2, R4, 0xfff); - __ StoreToOffset(kStoreHalfword, R2, R4, 0x1000); - __ StoreToOffset(kStoreHalfword, R2, R4, 0x1000a4); - __ StoreToOffset(kStoreHalfword, R2, R4, 0x101000); - __ StoreToOffset(kStoreHalfword, R4, R4, 0x101000); - __ StoreToOffset(kStoreWordPair, R2, R4, 12); - __ StoreToOffset(kStoreWordPair, R2, R4, 0x3fc); - __ StoreToOffset(kStoreWordPair, R2, R4, 0x400); - __ StoreToOffset(kStoreWordPair, R2, R4, 0x400a4); - __ StoreToOffset(kStoreWordPair, R2, R4, 0x40400); - __ StoreToOffset(kStoreWordPair, R4, R4, 0x40400); - - __ StoreToOffset(kStoreWord, R0, R12, 12); // 32-bit because of R12. - __ StoreToOffset(kStoreWord, R2, R4, 0xa4 - 0x100000); - - __ StoreToOffset(kStoreByte, R2, R4, 12); - - EmitAndCheck(&assembler, "StoreToOffset"); -} - -TEST_F(Thumb2AssemblerTest, IfThen) { - __ it(EQ); - __ mov(R1, ShifterOperand(1), EQ); - - __ it(EQ, kItThen); - __ mov(R1, ShifterOperand(1), EQ); - __ mov(R2, ShifterOperand(2), EQ); - - __ it(EQ, kItElse); - __ mov(R1, ShifterOperand(1), EQ); - __ mov(R2, ShifterOperand(2), NE); - - __ it(EQ, kItThen, kItElse); - __ mov(R1, ShifterOperand(1), EQ); - __ mov(R2, ShifterOperand(2), EQ); - __ mov(R3, ShifterOperand(3), NE); - - __ it(EQ, kItElse, kItElse); - __ mov(R1, ShifterOperand(1), EQ); - __ mov(R2, ShifterOperand(2), NE); - __ mov(R3, ShifterOperand(3), NE); - - __ it(EQ, kItThen, kItThen, kItElse); - __ mov(R1, ShifterOperand(1), EQ); - __ mov(R2, ShifterOperand(2), EQ); - __ mov(R3, ShifterOperand(3), EQ); - __ mov(R4, ShifterOperand(4), NE); - - EmitAndCheck(&assembler, "IfThen"); -} - -TEST_F(Thumb2AssemblerTest, CbzCbnz) { - Label l1; - __ cbz(R2, &l1); - __ mov(R1, ShifterOperand(3)); - __ mov(R2, ShifterOperand(3)); - __ Bind(&l1); - __ mov(R2, ShifterOperand(4)); - - Label l2; - __ cbnz(R2, &l2); - __ mov(R8, ShifterOperand(3)); - __ mov(R2, ShifterOperand(3)); - __ Bind(&l2); - __ mov(R2, ShifterOperand(4)); - - EmitAndCheck(&assembler, "CbzCbnz"); -} - -TEST_F(Thumb2AssemblerTest, Multiply) { - __ mul(R0, R1, R0); - __ mul(R0, R1, R2); - __ mul(R8, R9, R8); - __ mul(R8, R9, R10); - - __ mla(R0, R1, R2, R3); - __ mla(R8, R9, R8, R9); - - __ mls(R0, R1, R2, R3); - __ mls(R8, R9, R8, R9); - - __ umull(R0, R1, R2, R3); - __ umull(R8, R9, R10, R11); - - EmitAndCheck(&assembler, "Multiply"); -} - -TEST_F(Thumb2AssemblerTest, Divide) { - __ sdiv(R0, R1, R2); - __ sdiv(R8, R9, R10); - - __ udiv(R0, R1, R2); - __ udiv(R8, R9, R10); - - EmitAndCheck(&assembler, "Divide"); -} - -TEST_F(Thumb2AssemblerTest, VMov) { - __ vmovs(S1, 1.0); - __ vmovd(D1, 1.0); - - __ vmovs(S1, S2); - __ vmovd(D1, D2); - - EmitAndCheck(&assembler, "VMov"); -} - - -TEST_F(Thumb2AssemblerTest, BasicFloatingPoint) { - __ vadds(S0, S1, S2); - __ vsubs(S0, S1, S2); - __ vmuls(S0, S1, S2); - __ vmlas(S0, S1, S2); - __ vmlss(S0, S1, S2); - __ vdivs(S0, S1, S2); - __ vabss(S0, S1); - __ vnegs(S0, S1); - __ vsqrts(S0, S1); - - __ vaddd(D0, D1, D2); - __ vsubd(D0, D1, D2); - __ vmuld(D0, D1, D2); - __ vmlad(D0, D1, D2); - __ vmlsd(D0, D1, D2); - __ vdivd(D0, D1, D2); - __ vabsd(D0, D1); - __ vnegd(D0, D1); - __ vsqrtd(D0, D1); - - EmitAndCheck(&assembler, "BasicFloatingPoint"); -} - -TEST_F(Thumb2AssemblerTest, FloatingPointConversions) { - __ vcvtsd(S2, D2); - __ vcvtds(D2, S2); - - __ vcvtis(S1, S2); - __ vcvtsi(S1, S2); - - __ vcvtid(S1, D2); - __ vcvtdi(D1, S2); - - __ vcvtus(S1, S2); - __ vcvtsu(S1, S2); - - __ vcvtud(S1, D2); - __ vcvtdu(D1, S2); - - EmitAndCheck(&assembler, "FloatingPointConversions"); -} - -TEST_F(Thumb2AssemblerTest, FloatingPointComparisons) { - __ vcmps(S0, S1); - __ vcmpd(D0, D1); - - __ vcmpsz(S2); - __ vcmpdz(D2); - - EmitAndCheck(&assembler, "FloatingPointComparisons"); -} - -TEST_F(Thumb2AssemblerTest, Calls) { - __ blx(LR); - __ bx(LR); - - EmitAndCheck(&assembler, "Calls"); -} - -TEST_F(Thumb2AssemblerTest, Breakpoint) { - __ bkpt(0); - - EmitAndCheck(&assembler, "Breakpoint"); -} - -TEST_F(Thumb2AssemblerTest, StrR1) { - __ str(R1, Address(SP, 68)); - __ str(R1, Address(SP, 1068)); - - EmitAndCheck(&assembler, "StrR1"); -} - -TEST_F(Thumb2AssemblerTest, VPushPop) { - __ vpushs(S2, 4); - __ vpushd(D2, 4); - - __ vpops(S2, 4); - __ vpopd(D2, 4); - - EmitAndCheck(&assembler, "VPushPop"); -} - -TEST_F(Thumb2AssemblerTest, Max16BitBranch) { - Label l1; - __ b(&l1); - for (int i = 0 ; i < (1 << 11) ; i += 2) { - __ mov(R3, ShifterOperand(i & 0xff)); - } - __ Bind(&l1); - __ mov(R1, ShifterOperand(R2)); - - EmitAndCheck(&assembler, "Max16BitBranch"); -} - -TEST_F(Thumb2AssemblerTest, Branch32) { - Label l1; - __ b(&l1); - for (int i = 0 ; i < (1 << 11) + 2 ; i += 2) { - __ mov(R3, ShifterOperand(i & 0xff)); - } - __ Bind(&l1); - __ mov(R1, ShifterOperand(R2)); - - EmitAndCheck(&assembler, "Branch32"); -} - -TEST_F(Thumb2AssemblerTest, CompareAndBranchMax) { - Label l1; - __ cbz(R4, &l1); - for (int i = 0 ; i < (1 << 7) ; i += 2) { - __ mov(R3, ShifterOperand(i & 0xff)); - } - __ Bind(&l1); - __ mov(R1, ShifterOperand(R2)); - - EmitAndCheck(&assembler, "CompareAndBranchMax"); -} - -TEST_F(Thumb2AssemblerTest, CompareAndBranchRelocation16) { - Label l1; - __ cbz(R4, &l1); - for (int i = 0 ; i < (1 << 7) + 2 ; i += 2) { - __ mov(R3, ShifterOperand(i & 0xff)); - } - __ Bind(&l1); - __ mov(R1, ShifterOperand(R2)); - - EmitAndCheck(&assembler, "CompareAndBranchRelocation16"); -} - -TEST_F(Thumb2AssemblerTest, CompareAndBranchRelocation32) { - Label l1; - __ cbz(R4, &l1); - for (int i = 0 ; i < (1 << 11) + 2 ; i += 2) { - __ mov(R3, ShifterOperand(i & 0xff)); - } - __ Bind(&l1); - __ mov(R1, ShifterOperand(R2)); - - EmitAndCheck(&assembler, "CompareAndBranchRelocation32"); -} - -TEST_F(Thumb2AssemblerTest, MixedBranch32) { - Label l1; - Label l2; - __ b(&l1); // Forwards. - __ Bind(&l2); - - // Space to force relocation. - for (int i = 0 ; i < (1 << 11) + 2 ; i += 2) { - __ mov(R3, ShifterOperand(i & 0xff)); - } - __ b(&l2); // Backwards. - __ Bind(&l1); - __ mov(R1, ShifterOperand(R2)); - - EmitAndCheck(&assembler, "MixedBranch32"); -} - -TEST_F(Thumb2AssemblerTest, Shifts) { - // 16 bit selected for CcDontCare. - __ Lsl(R0, R1, 5); - __ Lsr(R0, R1, 5); - __ Asr(R0, R1, 5); - - __ Lsl(R0, R0, R1); - __ Lsr(R0, R0, R1); - __ Asr(R0, R0, R1); - __ Ror(R0, R0, R1); - - // 16 bit with kCcSet. - __ Lsls(R0, R1, 5); - __ Lsrs(R0, R1, 5); - __ Asrs(R0, R1, 5); - - __ Lsls(R0, R0, R1); - __ Lsrs(R0, R0, R1); - __ Asrs(R0, R0, R1); - __ Rors(R0, R0, R1); - - // 32-bit with kCcKeep. - __ Lsl(R0, R1, 5, AL, kCcKeep); - __ Lsr(R0, R1, 5, AL, kCcKeep); - __ Asr(R0, R1, 5, AL, kCcKeep); - - __ Lsl(R0, R0, R1, AL, kCcKeep); - __ Lsr(R0, R0, R1, AL, kCcKeep); - __ Asr(R0, R0, R1, AL, kCcKeep); - __ Ror(R0, R0, R1, AL, kCcKeep); - - // 32-bit because ROR immediate doesn't have a 16-bit version like the other shifts. - __ Ror(R0, R1, 5); - __ Rors(R0, R1, 5); - __ Ror(R0, R1, 5, AL, kCcKeep); - - // 32 bit due to high registers. - __ Lsl(R8, R1, 5); - __ Lsr(R0, R8, 5); - __ Asr(R8, R1, 5); - __ Ror(R0, R8, 5); - - // 32 bit due to different Rd and Rn. - __ Lsl(R0, R1, R2); - __ Lsr(R0, R1, R2); - __ Asr(R0, R1, R2); - __ Ror(R0, R1, R2); - - // 32 bit due to use of high registers. - __ Lsl(R8, R1, R2); - __ Lsr(R0, R8, R2); - __ Asr(R0, R1, R8); - - // S bit (all 32 bit) - - // 32 bit due to high registers. - __ Lsls(R8, R1, 5); - __ Lsrs(R0, R8, 5); - __ Asrs(R8, R1, 5); - __ Rors(R0, R8, 5); - - // 32 bit due to different Rd and Rn. - __ Lsls(R0, R1, R2); - __ Lsrs(R0, R1, R2); - __ Asrs(R0, R1, R2); - __ Rors(R0, R1, R2); - - // 32 bit due to use of high registers. - __ Lsls(R8, R1, R2); - __ Lsrs(R0, R8, R2); - __ Asrs(R0, R1, R8); - - EmitAndCheck(&assembler, "Shifts"); -} - -TEST_F(Thumb2AssemblerTest, LoadStoreRegOffset) { - // 16 bit. - __ ldr(R0, Address(R1, R2)); - __ str(R0, Address(R1, R2)); - - // 32 bit due to shift. - __ ldr(R0, Address(R1, R2, LSL, 1)); - __ str(R0, Address(R1, R2, LSL, 1)); - - __ ldr(R0, Address(R1, R2, LSL, 3)); - __ str(R0, Address(R1, R2, LSL, 3)); - - // 32 bit due to high register use. - __ ldr(R8, Address(R1, R2)); - __ str(R8, Address(R1, R2)); - - __ ldr(R1, Address(R8, R2)); - __ str(R2, Address(R8, R2)); - - __ ldr(R0, Address(R1, R8)); - __ str(R0, Address(R1, R8)); - - EmitAndCheck(&assembler, "LoadStoreRegOffset"); -} - -TEST_F(Thumb2AssemblerTest, LoadStoreLimits) { - __ ldr(R0, Address(R4, 124)); // 16 bit. - __ ldr(R0, Address(R4, 128)); // 32 bit. - - __ ldrb(R0, Address(R4, 31)); // 16 bit. - __ ldrb(R0, Address(R4, 32)); // 32 bit. - - __ ldrh(R0, Address(R4, 62)); // 16 bit. - __ ldrh(R0, Address(R4, 64)); // 32 bit. - - __ ldrsb(R0, Address(R4, 31)); // 32 bit. - __ ldrsb(R0, Address(R4, 32)); // 32 bit. - - __ ldrsh(R0, Address(R4, 62)); // 32 bit. - __ ldrsh(R0, Address(R4, 64)); // 32 bit. - - __ str(R0, Address(R4, 124)); // 16 bit. - __ str(R0, Address(R4, 128)); // 32 bit. - - __ strb(R0, Address(R4, 31)); // 16 bit. - __ strb(R0, Address(R4, 32)); // 32 bit. - - __ strh(R0, Address(R4, 62)); // 16 bit. - __ strh(R0, Address(R4, 64)); // 32 bit. - - EmitAndCheck(&assembler, "LoadStoreLimits"); -} - -TEST_F(Thumb2AssemblerTest, CompareAndBranch) { - Label label; - __ CompareAndBranchIfZero(arm::R0, &label); - __ CompareAndBranchIfZero(arm::R11, &label); - __ CompareAndBranchIfNonZero(arm::R0, &label); - __ CompareAndBranchIfNonZero(arm::R11, &label); - __ Bind(&label); - - EmitAndCheck(&assembler, "CompareAndBranch"); -} - -TEST_F(Thumb2AssemblerTest, AddConstant) { - // Low registers, Rd != Rn. - __ AddConstant(R0, R1, 0); // MOV. - __ AddConstant(R0, R1, 1); // 16-bit ADDS, encoding T1. - __ AddConstant(R0, R1, 7); // 16-bit ADDS, encoding T1. - __ AddConstant(R0, R1, 8); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 255); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 256); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 257); // 32-bit ADD, encoding T4. - __ AddConstant(R0, R1, 0xfff); // 32-bit ADD, encoding T4. - __ AddConstant(R0, R1, 0x1000); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 0x1001); // MVN+SUB. - __ AddConstant(R0, R1, 0x1002); // MOVW+ADD. - __ AddConstant(R0, R1, 0xffff); // MOVW+ADD. - __ AddConstant(R0, R1, 0x10000); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 0x10001); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 0x10002); // MVN+SUB. - __ AddConstant(R0, R1, 0x10003); // MOVW+MOVT+ADD. - __ AddConstant(R0, R1, -1); // 16-bit SUBS. - __ AddConstant(R0, R1, -7); // 16-bit SUBS. - __ AddConstant(R0, R1, -8); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -255); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -256); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -257); // 32-bit SUB, encoding T4. - __ AddConstant(R0, R1, -0xfff); // 32-bit SUB, encoding T4. - __ AddConstant(R0, R1, -0x1000); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -0x1001); // MVN+ADD. - __ AddConstant(R0, R1, -0x1002); // MOVW+SUB. - __ AddConstant(R0, R1, -0xffff); // MOVW+SUB. - __ AddConstant(R0, R1, -0x10000); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -0x10001); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -0x10002); // MVN+ADD. - __ AddConstant(R0, R1, -0x10003); // MOVW+MOVT+ADD. - - // Low registers, Rd == Rn. - __ AddConstant(R0, R0, 0); // Nothing. - __ AddConstant(R1, R1, 1); // 16-bit ADDS, encoding T2, - __ AddConstant(R0, R0, 7); // 16-bit ADDS, encoding T2. - __ AddConstant(R1, R1, 8); // 16-bit ADDS, encoding T2. - __ AddConstant(R0, R0, 255); // 16-bit ADDS, encoding T2. - __ AddConstant(R1, R1, 256); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R0, 257); // 32-bit ADD, encoding T4. - __ AddConstant(R1, R1, 0xfff); // 32-bit ADD, encoding T4. - __ AddConstant(R0, R0, 0x1000); // 32-bit ADD, encoding T3. - __ AddConstant(R1, R1, 0x1001); // MVN+SUB. - __ AddConstant(R0, R0, 0x1002); // MOVW+ADD. - __ AddConstant(R1, R1, 0xffff); // MOVW+ADD. - __ AddConstant(R0, R0, 0x10000); // 32-bit ADD, encoding T3. - __ AddConstant(R1, R1, 0x10001); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R0, 0x10002); // MVN+SUB. - __ AddConstant(R1, R1, 0x10003); // MOVW+MOVT+ADD. - __ AddConstant(R0, R0, -1); // 16-bit SUBS, encoding T2. - __ AddConstant(R1, R1, -7); // 16-bit SUBS, encoding T2. - __ AddConstant(R0, R0, -8); // 16-bit SUBS, encoding T2. - __ AddConstant(R1, R1, -255); // 16-bit SUBS, encoding T2. - __ AddConstant(R0, R0, -256); // 32-bit SUB, encoding T3. - __ AddConstant(R1, R1, -257); // 32-bit SUB, encoding T4. - __ AddConstant(R0, R0, -0xfff); // 32-bit SUB, encoding T4. - __ AddConstant(R1, R1, -0x1000); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R0, -0x1001); // MVN+ADD. - __ AddConstant(R1, R1, -0x1002); // MOVW+SUB. - __ AddConstant(R0, R0, -0xffff); // MOVW+SUB. - __ AddConstant(R1, R1, -0x10000); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R0, -0x10001); // 32-bit SUB, encoding T3. - __ AddConstant(R1, R1, -0x10002); // MVN+ADD. - __ AddConstant(R0, R0, -0x10003); // MOVW+MOVT+ADD. - - // High registers. - __ AddConstant(R8, R8, 0); // Nothing. - __ AddConstant(R8, R1, 1); // 32-bit ADD, encoding T3, - __ AddConstant(R0, R8, 7); // 32-bit ADD, encoding T3. - __ AddConstant(R8, R8, 8); // 32-bit ADD, encoding T3. - __ AddConstant(R8, R1, 255); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R8, 256); // 32-bit ADD, encoding T3. - __ AddConstant(R8, R8, 257); // 32-bit ADD, encoding T4. - __ AddConstant(R8, R1, 0xfff); // 32-bit ADD, encoding T4. - __ AddConstant(R0, R8, 0x1000); // 32-bit ADD, encoding T3. - __ AddConstant(R8, R8, 0x1001); // MVN+SUB. - __ AddConstant(R0, R1, 0x1002); // MOVW+ADD. - __ AddConstant(R0, R8, 0xffff); // MOVW+ADD. - __ AddConstant(R8, R8, 0x10000); // 32-bit ADD, encoding T3. - __ AddConstant(R8, R1, 0x10001); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R8, 0x10002); // MVN+SUB. - __ AddConstant(R0, R8, 0x10003); // MOVW+MOVT+ADD. - __ AddConstant(R8, R8, -1); // 32-bit ADD, encoding T3. - __ AddConstant(R8, R1, -7); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R8, -8); // 32-bit SUB, encoding T3. - __ AddConstant(R8, R8, -255); // 32-bit SUB, encoding T3. - __ AddConstant(R8, R1, -256); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R8, -257); // 32-bit SUB, encoding T4. - __ AddConstant(R8, R8, -0xfff); // 32-bit SUB, encoding T4. - __ AddConstant(R8, R1, -0x1000); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R8, -0x1001); // MVN+ADD. - __ AddConstant(R0, R1, -0x1002); // MOVW+SUB. - __ AddConstant(R8, R1, -0xffff); // MOVW+SUB. - __ AddConstant(R0, R8, -0x10000); // 32-bit SUB, encoding T3. - __ AddConstant(R8, R8, -0x10001); // 32-bit SUB, encoding T3. - __ AddConstant(R8, R1, -0x10002); // MVN+SUB. - __ AddConstant(R0, R8, -0x10003); // MOVW+MOVT+ADD. - - // Low registers, Rd != Rn, kCcKeep. - __ AddConstant(R0, R1, 0, AL, kCcKeep); // MOV. - __ AddConstant(R0, R1, 1, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 7, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 8, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 255, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 256, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 257, AL, kCcKeep); // 32-bit ADD, encoding T4. - __ AddConstant(R0, R1, 0xfff, AL, kCcKeep); // 32-bit ADD, encoding T4. - __ AddConstant(R0, R1, 0x1000, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 0x1001, AL, kCcKeep); // MVN+SUB. - __ AddConstant(R0, R1, 0x1002, AL, kCcKeep); // MOVW+ADD. - __ AddConstant(R0, R1, 0xffff, AL, kCcKeep); // MOVW+ADD. - __ AddConstant(R0, R1, 0x10000, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 0x10001, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, 0x10002, AL, kCcKeep); // MVN+SUB. - __ AddConstant(R0, R1, 0x10003, AL, kCcKeep); // MOVW+MOVT+ADD. - __ AddConstant(R0, R1, -1, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R1, -7, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -8, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -255, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -256, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -257, AL, kCcKeep); // 32-bit SUB, encoding T4. - __ AddConstant(R0, R1, -0xfff, AL, kCcKeep); // 32-bit SUB, encoding T4. - __ AddConstant(R0, R1, -0x1000, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -0x1001, AL, kCcKeep); // MVN+ADD. - __ AddConstant(R0, R1, -0x1002, AL, kCcKeep); // MOVW+SUB. - __ AddConstant(R0, R1, -0xffff, AL, kCcKeep); // MOVW+SUB. - __ AddConstant(R0, R1, -0x10000, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -0x10001, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R1, -0x10002, AL, kCcKeep); // MVN+ADD. - __ AddConstant(R0, R1, -0x10003, AL, kCcKeep); // MOVW+MOVT+ADD. - - // Low registers, Rd == Rn, kCcKeep. - __ AddConstant(R0, R0, 0, AL, kCcKeep); // Nothing. - __ AddConstant(R1, R1, 1, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R0, 7, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R1, R1, 8, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R0, 255, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R1, R1, 256, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R0, 257, AL, kCcKeep); // 32-bit ADD, encoding T4. - __ AddConstant(R1, R1, 0xfff, AL, kCcKeep); // 32-bit ADD, encoding T4. - __ AddConstant(R0, R0, 0x1000, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R1, R1, 0x1001, AL, kCcKeep); // MVN+SUB. - __ AddConstant(R0, R0, 0x1002, AL, kCcKeep); // MOVW+ADD. - __ AddConstant(R1, R1, 0xffff, AL, kCcKeep); // MOVW+ADD. - __ AddConstant(R0, R0, 0x10000, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R1, R1, 0x10001, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R0, R0, 0x10002, AL, kCcKeep); // MVN+SUB. - __ AddConstant(R1, R1, 0x10003, AL, kCcKeep); // MOVW+MOVT+ADD. - __ AddConstant(R0, R0, -1, AL, kCcKeep); // 32-bit ADD, encoding T3. - __ AddConstant(R1, R1, -7, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R0, -8, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R1, R1, -255, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R0, -256, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R1, R1, -257, AL, kCcKeep); // 32-bit SUB, encoding T4. - __ AddConstant(R0, R0, -0xfff, AL, kCcKeep); // 32-bit SUB, encoding T4. - __ AddConstant(R1, R1, -0x1000, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R0, -0x1001, AL, kCcKeep); // MVN+ADD. - __ AddConstant(R1, R1, -0x1002, AL, kCcKeep); // MOVW+SUB. - __ AddConstant(R0, R0, -0xffff, AL, kCcKeep); // MOVW+SUB. - __ AddConstant(R1, R1, -0x10000, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R0, -0x10001, AL, kCcKeep); // 32-bit SUB, encoding T3. - __ AddConstant(R1, R1, -0x10002, AL, kCcKeep); // MVN+ADD. - __ AddConstant(R0, R0, -0x10003, AL, kCcKeep); // MOVW+MOVT+ADD. - - // Low registers, Rd != Rn, kCcSet. - __ AddConstant(R0, R1, 0, AL, kCcSet); // 16-bit ADDS. - __ AddConstant(R0, R1, 1, AL, kCcSet); // 16-bit ADDS. - __ AddConstant(R0, R1, 7, AL, kCcSet); // 16-bit ADDS. - __ AddConstant(R0, R1, 8, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R1, 255, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R1, 256, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R1, 257, AL, kCcSet); // MVN+SUBS. - __ AddConstant(R0, R1, 0xfff, AL, kCcSet); // MOVW+ADDS. - __ AddConstant(R0, R1, 0x1000, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R1, 0x1001, AL, kCcSet); // MVN+SUBS. - __ AddConstant(R0, R1, 0x1002, AL, kCcSet); // MOVW+ADDS. - __ AddConstant(R0, R1, 0xffff, AL, kCcSet); // MOVW+ADDS. - __ AddConstant(R0, R1, 0x10000, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R1, 0x10001, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R1, 0x10002, AL, kCcSet); // MVN+SUBS. - __ AddConstant(R0, R1, 0x10003, AL, kCcSet); // MOVW+MOVT+ADDS. - __ AddConstant(R0, R1, -1, AL, kCcSet); // 16-bit SUBS. - __ AddConstant(R0, R1, -7, AL, kCcSet); // 16-bit SUBS. - __ AddConstant(R0, R1, -8, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R0, R1, -255, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R0, R1, -256, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R0, R1, -257, AL, kCcSet); // MVN+ADDS. - __ AddConstant(R0, R1, -0xfff, AL, kCcSet); // MOVW+SUBS. - __ AddConstant(R0, R1, -0x1000, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R0, R1, -0x1001, AL, kCcSet); // MVN+ADDS. - __ AddConstant(R0, R1, -0x1002, AL, kCcSet); // MOVW+SUBS. - __ AddConstant(R0, R1, -0xffff, AL, kCcSet); // MOVW+SUBS. - __ AddConstant(R0, R1, -0x10000, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R0, R1, -0x10001, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R0, R1, -0x10002, AL, kCcSet); // MVN+ADDS. - __ AddConstant(R0, R1, -0x10003, AL, kCcSet); // MOVW+MOVT+ADDS. - - // Low registers, Rd == Rn, kCcSet. - __ AddConstant(R0, R0, 0, AL, kCcSet); // 16-bit ADDS, encoding T2. - __ AddConstant(R1, R1, 1, AL, kCcSet); // 16-bit ADDS, encoding T2. - __ AddConstant(R0, R0, 7, AL, kCcSet); // 16-bit ADDS, encoding T2. - __ AddConstant(R1, R1, 8, AL, kCcSet); // 16-bit ADDS, encoding T2. - __ AddConstant(R0, R0, 255, AL, kCcSet); // 16-bit ADDS, encoding T2. - __ AddConstant(R1, R1, 256, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R0, 257, AL, kCcSet); // MVN+SUBS. - __ AddConstant(R1, R1, 0xfff, AL, kCcSet); // MOVW+ADDS. - __ AddConstant(R0, R0, 0x1000, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R1, R1, 0x1001, AL, kCcSet); // MVN+SUBS. - __ AddConstant(R0, R0, 0x1002, AL, kCcSet); // MOVW+ADDS. - __ AddConstant(R1, R1, 0xffff, AL, kCcSet); // MOVW+ADDS. - __ AddConstant(R0, R0, 0x10000, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R1, R1, 0x10001, AL, kCcSet); // 32-bit ADDS, encoding T3. - __ AddConstant(R0, R0, 0x10002, AL, kCcSet); // MVN+SUBS. - __ AddConstant(R1, R1, 0x10003, AL, kCcSet); // MOVW+MOVT+ADDS. - __ AddConstant(R0, R0, -1, AL, kCcSet); // 16-bit SUBS, encoding T2. - __ AddConstant(R1, R1, -7, AL, kCcSet); // 16-bit SUBS, encoding T2. - __ AddConstant(R0, R0, -8, AL, kCcSet); // 16-bit SUBS, encoding T2. - __ AddConstant(R1, R1, -255, AL, kCcSet); // 16-bit SUBS, encoding T2. - __ AddConstant(R0, R0, -256, AL, kCcSet); // 32-bit SUB, encoding T3. - __ AddConstant(R1, R1, -257, AL, kCcSet); // MNV+ADDS. - __ AddConstant(R0, R0, -0xfff, AL, kCcSet); // MOVW+SUBS. - __ AddConstant(R1, R1, -0x1000, AL, kCcSet); // 32-bit SUB, encoding T3. - __ AddConstant(R0, R0, -0x1001, AL, kCcSet); // MVN+ADDS. - __ AddConstant(R1, R1, -0x1002, AL, kCcSet); // MOVW+SUBS. - __ AddConstant(R0, R0, -0xffff, AL, kCcSet); // MOVW+SUBS. - __ AddConstant(R1, R1, -0x10000, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R0, R0, -0x10001, AL, kCcSet); // 32-bit SUBS, encoding T3. - __ AddConstant(R1, R1, -0x10002, AL, kCcSet); // MVN+ADDS. - __ AddConstant(R0, R0, -0x10003, AL, kCcSet); // MOVW+MOVT+ADDS. - - __ it(EQ); - __ AddConstant(R0, R1, 1, EQ, kCcSet); // 32-bit ADDS, encoding T3. - __ it(NE); - __ AddConstant(R0, R1, 1, NE, kCcKeep); // 16-bit ADDS, encoding T1. - __ it(GE); - __ AddConstant(R0, R0, 1, GE, kCcSet); // 32-bit ADDS, encoding T3. - __ it(LE); - __ AddConstant(R0, R0, 1, LE, kCcKeep); // 16-bit ADDS, encoding T2. - - EmitAndCheck(&assembler, "AddConstant"); -} - -TEST_F(Thumb2AssemblerTest, CmpConstant) { - __ CmpConstant(R0, 0); // 16-bit CMP. - __ CmpConstant(R1, 1); // 16-bit CMP. - __ CmpConstant(R0, 7); // 16-bit CMP. - __ CmpConstant(R1, 8); // 16-bit CMP. - __ CmpConstant(R0, 255); // 16-bit CMP. - __ CmpConstant(R1, 256); // 32-bit CMP. - __ CmpConstant(R0, 257); // MNV+CMN. - __ CmpConstant(R1, 0xfff); // MOVW+CMP. - __ CmpConstant(R0, 0x1000); // 32-bit CMP. - __ CmpConstant(R1, 0x1001); // MNV+CMN. - __ CmpConstant(R0, 0x1002); // MOVW+CMP. - __ CmpConstant(R1, 0xffff); // MOVW+CMP. - __ CmpConstant(R0, 0x10000); // 32-bit CMP. - __ CmpConstant(R1, 0x10001); // 32-bit CMP. - __ CmpConstant(R0, 0x10002); // MVN+CMN. - __ CmpConstant(R1, 0x10003); // MOVW+MOVT+CMP. - __ CmpConstant(R0, -1); // 32-bit CMP. - __ CmpConstant(R1, -7); // CMN. - __ CmpConstant(R0, -8); // CMN. - __ CmpConstant(R1, -255); // CMN. - __ CmpConstant(R0, -256); // CMN. - __ CmpConstant(R1, -257); // MNV+CMP. - __ CmpConstant(R0, -0xfff); // MOVW+CMN. - __ CmpConstant(R1, -0x1000); // CMN. - __ CmpConstant(R0, -0x1001); // MNV+CMP. - __ CmpConstant(R1, -0x1002); // MOVW+CMN. - __ CmpConstant(R0, -0xffff); // MOVW+CMN. - __ CmpConstant(R1, -0x10000); // CMN. - __ CmpConstant(R0, -0x10001); // CMN. - __ CmpConstant(R1, -0x10002); // MVN+CMP. - __ CmpConstant(R0, -0x10003); // MOVW+MOVT+CMP. - - __ CmpConstant(R8, 0); // 32-bit CMP. - __ CmpConstant(R9, 1); // 32-bit CMP. - __ CmpConstant(R8, 7); // 32-bit CMP. - __ CmpConstant(R9, 8); // 32-bit CMP. - __ CmpConstant(R8, 255); // 32-bit CMP. - __ CmpConstant(R9, 256); // 32-bit CMP. - __ CmpConstant(R8, 257); // MNV+CMN - __ CmpConstant(R9, 0xfff); // MOVW+CMP. - __ CmpConstant(R8, 0x1000); // 32-bit CMP. - __ CmpConstant(R9, 0x1001); // MVN+CMN. - __ CmpConstant(R8, 0x1002); // MOVW+CMP. - __ CmpConstant(R9, 0xffff); // MOVW+CMP. - __ CmpConstant(R8, 0x10000); // 32-bit CMP. - __ CmpConstant(R9, 0x10001); // 32-bit CMP. - __ CmpConstant(R8, 0x10002); // MVN+CMN. - __ CmpConstant(R9, 0x10003); // MOVW+MOVT+CMP. - __ CmpConstant(R8, -1); // 32-bit CMP - __ CmpConstant(R9, -7); // CMN. - __ CmpConstant(R8, -8); // CMN. - __ CmpConstant(R9, -255); // CMN. - __ CmpConstant(R8, -256); // CMN. - __ CmpConstant(R9, -257); // MNV+CMP. - __ CmpConstant(R8, -0xfff); // MOVW+CMN. - __ CmpConstant(R9, -0x1000); // CMN. - __ CmpConstant(R8, -0x1001); // MVN+CMP. - __ CmpConstant(R9, -0x1002); // MOVW+CMN. - __ CmpConstant(R8, -0xffff); // MOVW+CMN. - __ CmpConstant(R9, -0x10000); // CMN. - __ CmpConstant(R8, -0x10001); // CMN. - __ CmpConstant(R9, -0x10002); // MVN+CMP. - __ CmpConstant(R8, -0x10003); // MOVW+MOVT+CMP. - - EmitAndCheck(&assembler, "CmpConstant"); -} - -#define ENABLE_VIXL_TEST - -#ifdef ENABLE_VIXL_TEST - -#define ARM_VIXL - -#ifdef ARM_VIXL -typedef arm::ArmVIXLJNIMacroAssembler JniAssemblerType; -#else -typedef arm::Thumb2Assembler AssemblerType; -#endif - class ArmVIXLAssemblerTest : public ::testing::Test { public: ArmVIXLAssemblerTest() : pool(), arena(&pool), assembler(&arena) { } ArenaPool pool; ArenaAllocator arena; - JniAssemblerType assembler; + ArmVIXLJNIMacroAssembler assembler; }; -#undef __ #define __ assembler-> -void EmitAndCheck(JniAssemblerType* assembler, const char* testname, +void EmitAndCheck(ArmVIXLJNIMacroAssembler* assembler, const char* testname, const char* const* results) { __ FinalizeCode(); size_t cs = __ CodeSize(); @@ -1631,7 +197,7 @@ void EmitAndCheck(JniAssemblerType* assembler, const char* testname, DumpAndCheck(managed_code, testname, results); } -void EmitAndCheck(JniAssemblerType* assembler, const char* testname) { +void EmitAndCheck(ArmVIXLJNIMacroAssembler* assembler, const char* testname) { InitResults(); std::map<std::string, const char* const*>::iterator results = test_results.find(testname); ASSERT_NE(results, test_results.end()); @@ -1640,9 +206,14 @@ void EmitAndCheck(JniAssemblerType* assembler, const char* testname) { } #undef __ + #define __ assembler. TEST_F(ArmVIXLAssemblerTest, VixlJniHelpers) { + // Run the test only with Baker read barriers, as the expected + // generated code contains a Marking Register refresh instruction. + TEST_DISABLED_WITHOUT_BAKER_READ_BARRIERS(); + const bool is_static = true; const bool is_synchronized = false; const bool is_critical_native = false; @@ -1729,14 +300,15 @@ TEST_F(ArmVIXLAssemblerTest, VixlJniHelpers) { EmitAndCheck(&assembler, "VixlJniHelpers"); } -#ifdef ARM_VIXL +#undef __ + +// TODO: Avoid these macros. #define R0 vixl::aarch32::r0 #define R2 vixl::aarch32::r2 #define R4 vixl::aarch32::r4 #define R12 vixl::aarch32::r12 -#undef __ + #define __ assembler.asm_. -#endif TEST_F(ArmVIXLAssemblerTest, VixlLoadFromOffset) { __ LoadFromOffset(kLoadWord, R2, R4, 12); @@ -1803,6 +375,5 @@ TEST_F(ArmVIXLAssemblerTest, VixlStoreToOffset) { } #undef __ -#endif // ENABLE_VIXL_TEST } // namespace arm } // namespace art diff --git a/compiler/utils/assembler_thumb_test_expected.cc.inc b/compiler/utils/assembler_thumb_test_expected.cc.inc index eaaf81518a..0a094352e4 100644 --- a/compiler/utils/assembler_thumb_test_expected.cc.inc +++ b/compiler/utils/assembler_thumb_test_expected.cc.inc @@ -1,5462 +1,3 @@ -const char* const SimpleMovResults[] = { - " 0: 0008 movs r0, r1\n", - " 2: 4608 mov r0, r1\n", - " 4: 46c8 mov r8, r9\n", - " 6: 2001 movs r0, #1\n", - " 8: f04f 0809 mov.w r8, #9\n", - nullptr -}; -const char* const SimpleMov32Results[] = { - " 0: ea4f 0001 mov.w r0, r1\n", - " 4: ea4f 0809 mov.w r8, r9\n", - nullptr -}; -const char* const SimpleMovAddResults[] = { - " 0: 4608 mov r0, r1\n", - " 2: 1888 adds r0, r1, r2\n", - " 4: 1c08 adds r0, r1, #0\n", - nullptr -}; -const char* const DataProcessingRegisterResults[] = { - " 0: ea6f 0001 mvn.w r0, r1\n", - " 4: eb01 0002 add.w r0, r1, r2\n", - " 8: eba1 0002 sub.w r0, r1, r2\n", - " c: ea01 0002 and.w r0, r1, r2\n", - " 10: ea41 0002 orr.w r0, r1, r2\n", - " 14: ea61 0002 orn r0, r1, r2\n", - " 18: ea81 0002 eor.w r0, r1, r2\n", - " 1c: ea21 0002 bic.w r0, r1, r2\n", - " 20: eb41 0002 adc.w r0, r1, r2\n", - " 24: eb61 0002 sbc.w r0, r1, r2\n", - " 28: ebc1 0002 rsb r0, r1, r2\n", - " 2c: ea90 0f01 teq r0, r1\n", - " 30: 0008 movs r0, r1\n", - " 32: 4608 mov r0, r1\n", - " 34: 43c8 mvns r0, r1\n", - " 36: 4408 add r0, r1\n", - " 38: 1888 adds r0, r1, r2\n", - " 3a: 1a88 subs r0, r1, r2\n", - " 3c: 4148 adcs r0, r1\n", - " 3e: 4188 sbcs r0, r1\n", - " 40: 4008 ands r0, r1\n", - " 42: 4308 orrs r0, r1\n", - " 44: 4048 eors r0, r1\n", - " 46: 4388 bics r0, r1\n", - " 48: 4208 tst r0, r1\n", - " 4a: 4288 cmp r0, r1\n", - " 4c: 42c8 cmn r0, r1\n", - " 4e: 4641 mov r1, r8\n", - " 50: 4681 mov r9, r0\n", - " 52: 46c8 mov r8, r9\n", - " 54: 4441 add r1, r8\n", - " 56: 4481 add r9, r0\n", - " 58: 44c8 add r8, r9\n", - " 5a: 4548 cmp r0, r9\n", - " 5c: 4588 cmp r8, r1\n", - " 5e: 45c1 cmp r9, r8\n", - " 60: 4248 negs r0, r1\n", - " 62: 4240 negs r0, r0\n", - " 64: ea5f 0008 movs.w r0, r8\n", - " 68: ea7f 0008 mvns.w r0, r8\n", - " 6c: eb01 0008 add.w r0, r1, r8\n", - " 70: eb11 0008 adds.w r0, r1, r8\n", - " 74: ebb1 0008 subs.w r0, r1, r8\n", - " 78: eb50 0008 adcs.w r0, r0, r8\n", - " 7c: eb70 0008 sbcs.w r0, r0, r8\n", - " 80: ea10 0008 ands.w r0, r0, r8\n", - " 84: ea50 0008 orrs.w r0, r0, r8\n", - " 88: ea90 0008 eors.w r0, r0, r8\n", - " 8c: ea30 0008 bics.w r0, r0, r8\n", - " 90: ea10 0f08 tst.w r0, r8\n", - " 94: eb10 0f08 cmn.w r0, r8\n", - " 98: f1d8 0000 rsbs r0, r8, #0\n", - " 9c: f1d8 0800 rsbs r8, r8, #0\n", - " a0: bf08 it eq\n", - " a2: ea7f 0001 mvnseq.w r0, r1\n", - " a6: bf08 it eq\n", - " a8: eb11 0002 addseq.w r0, r1, r2\n", - " ac: bf08 it eq\n", - " ae: ebb1 0002 subseq.w r0, r1, r2\n", - " b2: bf08 it eq\n", - " b4: eb50 0001 adcseq.w r0, r0, r1\n", - " b8: bf08 it eq\n", - " ba: eb70 0001 sbcseq.w r0, r0, r1\n", - " be: bf08 it eq\n", - " c0: ea10 0001 andseq.w r0, r0, r1\n", - " c4: bf08 it eq\n", - " c6: ea50 0001 orrseq.w r0, r0, r1\n", - " ca: bf08 it eq\n", - " cc: ea90 0001 eorseq.w r0, r0, r1\n", - " d0: bf08 it eq\n", - " d2: ea30 0001 bicseq.w r0, r0, r1\n", - " d6: bf08 it eq\n", - " d8: 43c8 mvneq r0, r1\n", - " da: bf08 it eq\n", - " dc: 1888 addeq r0, r1, r2\n", - " de: bf08 it eq\n", - " e0: 1a88 subeq r0, r1, r2\n", - " e2: bf08 it eq\n", - " e4: 4148 adceq r0, r1\n", - " e6: bf08 it eq\n", - " e8: 4188 sbceq r0, r1\n", - " ea: bf08 it eq\n", - " ec: 4008 andeq r0, r1\n", - " ee: bf08 it eq\n", - " f0: 4308 orreq r0, r1\n", - " f2: bf08 it eq\n", - " f4: 4048 eoreq r0, r1\n", - " f6: bf08 it eq\n", - " f8: 4388 biceq r0, r1\n", - " fa: 4608 mov r0, r1\n", - " fc: 43c8 mvns r0, r1\n", - " fe: 4408 add r0, r1\n", - " 100: 1888 adds r0, r1, r2\n", - " 102: 1a88 subs r0, r1, r2\n", - " 104: 4148 adcs r0, r1\n", - " 106: 4188 sbcs r0, r1\n", - " 108: 4008 ands r0, r1\n", - " 10a: 4308 orrs r0, r1\n", - " 10c: 4048 eors r0, r1\n", - " 10e: 4388 bics r0, r1\n", - " 110: 4641 mov r1, r8\n", - " 112: 4681 mov r9, r0\n", - " 114: 46c8 mov r8, r9\n", - " 116: 4441 add r1, r8\n", - " 118: 4481 add r9, r0\n", - " 11a: 44c8 add r8, r9\n", - " 11c: 4248 negs r0, r1\n", - " 11e: 4240 negs r0, r0\n", - " 120: eb01 0c00 add.w ip, r1, r0\n", - nullptr -}; -const char* const DataProcessingImmediateResults[] = { - " 0: 2055 movs r0, #85 ; 0x55\n", - " 2: f06f 0055 mvn.w r0, #85 ; 0x55\n", - " 6: f101 0055 add.w r0, r1, #85 ; 0x55\n", - " a: f1a1 0055 sub.w r0, r1, #85 ; 0x55\n", - " e: f001 0055 and.w r0, r1, #85 ; 0x55\n", - " 12: f041 0055 orr.w r0, r1, #85 ; 0x55\n", - " 16: f061 0055 orn r0, r1, #85 ; 0x55\n", - " 1a: f081 0055 eor.w r0, r1, #85 ; 0x55\n", - " 1e: f021 0055 bic.w r0, r1, #85 ; 0x55\n", - " 22: f141 0055 adc.w r0, r1, #85 ; 0x55\n", - " 26: f161 0055 sbc.w r0, r1, #85 ; 0x55\n", - " 2a: f1c1 0055 rsb r0, r1, #85 ; 0x55\n", - " 2e: f010 0f55 tst.w r0, #85 ; 0x55\n", - " 32: f090 0f55 teq r0, #85 ; 0x55\n", - " 36: 2855 cmp r0, #85 ; 0x55\n", - " 38: f110 0f55 cmn.w r0, #85 ; 0x55\n", - " 3c: 1d48 adds r0, r1, #5\n", - " 3e: 1f48 subs r0, r1, #5\n", - " 40: 2055 movs r0, #85 ; 0x55\n", - " 42: f07f 0055 mvns.w r0, #85 ; 0x55\n", - " 46: 1d48 adds r0, r1, #5\n", - " 48: 1f48 subs r0, r1, #5\n", - nullptr -}; -const char* const DataProcessingModifiedImmediateResults[] = { - " 0: f04f 1055 mov.w r0, #5570645 ; 0x550055\n", - " 4: f06f 1055 mvn.w r0, #5570645 ; 0x550055\n", - " 8: f101 1055 add.w r0, r1, #5570645 ; 0x550055\n", - " c: f1a1 1055 sub.w r0, r1, #5570645 ; 0x550055\n", - " 10: f001 1055 and.w r0, r1, #5570645 ; 0x550055\n", - " 14: f041 1055 orr.w r0, r1, #5570645 ; 0x550055\n", - " 18: f061 1055 orn r0, r1, #5570645 ; 0x550055\n", - " 1c: f081 1055 eor.w r0, r1, #5570645 ; 0x550055\n", - " 20: f021 1055 bic.w r0, r1, #5570645 ; 0x550055\n", - " 24: f141 1055 adc.w r0, r1, #5570645 ; 0x550055\n", - " 28: f161 1055 sbc.w r0, r1, #5570645 ; 0x550055\n", - " 2c: f1c1 1055 rsb r0, r1, #5570645 ; 0x550055\n", - " 30: f010 1f55 tst.w r0, #5570645 ; 0x550055\n", - " 34: f090 1f55 teq r0, #5570645 ; 0x550055\n", - " 38: f1b0 1f55 cmp.w r0, #5570645 ; 0x550055\n", - " 3c: f110 1f55 cmn.w r0, #5570645 ; 0x550055\n", - nullptr -}; -const char* const DataProcessingModifiedImmediatesResults[] = { - " 0: f04f 1055 mov.w r0, #5570645 ; 0x550055\n", - " 4: f04f 2055 mov.w r0, #1426085120 ; 0x55005500\n", - " 8: f04f 3055 mov.w r0, #1431655765 ; 0x55555555\n", - " c: f04f 4055 mov.w r0, #3573547008 ; 0xd5000000\n", - " 10: f04f 40d4 mov.w r0, #1778384896 ; 0x6a000000\n", - " 14: f44f 7054 mov.w r0, #848 ; 0x350\n", - " 18: f44f 70d4 mov.w r0, #424 ; 0x1a8\n", - nullptr -}; -const char* const DataProcessingShiftedRegisterResults[] = { - " 0: 0123 lsls r3, r4, #4\n", - " 2: 0963 lsrs r3, r4, #5\n", - " 4: 11a3 asrs r3, r4, #6\n", - " 6: ea5f 13f4 movs.w r3, r4, ror #7\n", - " a: ea5f 0334 movs.w r3, r4, rrx\n", - " e: ea4f 1304 mov.w r3, r4, lsl #4\n", - " 12: ea4f 1354 mov.w r3, r4, lsr #5\n", - " 16: ea4f 13a4 mov.w r3, r4, asr #6\n", - " 1a: ea4f 13f4 mov.w r3, r4, ror #7\n", - " 1e: ea4f 0334 mov.w r3, r4, rrx\n", - " 22: ea5f 1804 movs.w r8, r4, lsl #4\n", - " 26: ea5f 1854 movs.w r8, r4, lsr #5\n", - " 2a: ea5f 18a4 movs.w r8, r4, asr #6\n", - " 2e: ea5f 18f4 movs.w r8, r4, ror #7\n", - " 32: ea5f 0834 movs.w r8, r4, rrx\n", - nullptr -}; -const char* const ShiftImmediateResults[] = { - " 0: 0123 lsls r3, r4, #4\n", - " 2: 0963 lsrs r3, r4, #5\n", - " 4: 11a3 asrs r3, r4, #6\n", - " 6: ea4f 13f4 mov.w r3, r4, ror #7\n", - " a: ea4f 0334 mov.w r3, r4, rrx\n", - " e: ea4f 1304 mov.w r3, r4, lsl #4\n", - " 12: ea4f 1354 mov.w r3, r4, lsr #5\n", - " 16: ea4f 13a4 mov.w r3, r4, asr #6\n", - " 1a: ea4f 13f4 mov.w r3, r4, ror #7\n", - " 1e: ea4f 0334 mov.w r3, r4, rrx\n", - " 22: ea5f 1804 movs.w r8, r4, lsl #4\n", - " 26: ea5f 1854 movs.w r8, r4, lsr #5\n", - " 2a: ea5f 18a4 movs.w r8, r4, asr #6\n", - " 2e: ea5f 18f4 movs.w r8, r4, ror #7\n", - " 32: ea5f 0834 movs.w r8, r4, rrx\n", - nullptr -}; -const char* const BasicLoadResults[] = { - " 0: 69a3 ldr r3, [r4, #24]\n", - " 2: 7e23 ldrb r3, [r4, #24]\n", - " 4: 8b23 ldrh r3, [r4, #24]\n", - " 6: f994 3018 ldrsb.w r3, [r4, #24]\n", - " a: f9b4 3018 ldrsh.w r3, [r4, #24]\n", - " e: 9b06 ldr r3, [sp, #24]\n", - " 10: f8d4 8018 ldr.w r8, [r4, #24]\n", - " 14: f894 8018 ldrb.w r8, [r4, #24]\n", - " 18: f8b4 8018 ldrh.w r8, [r4, #24]\n", - " 1c: f994 8018 ldrsb.w r8, [r4, #24]\n", - " 20: f9b4 8018 ldrsh.w r8, [r4, #24]\n", - nullptr -}; -const char* const BasicStoreResults[] = { - " 0: 61a3 str r3, [r4, #24]\n", - " 2: 7623 strb r3, [r4, #24]\n", - " 4: 8323 strh r3, [r4, #24]\n", - " 6: 9306 str r3, [sp, #24]\n", - " 8: f8c4 8018 str.w r8, [r4, #24]\n", - " c: f884 8018 strb.w r8, [r4, #24]\n", - " 10: f8a4 8018 strh.w r8, [r4, #24]\n", - nullptr -}; -const char* const ComplexLoadResults[] = { - " 0: 69a3 ldr r3, [r4, #24]\n", - " 2: f854 3f18 ldr.w r3, [r4, #24]!\n", - " 6: f854 3b18 ldr.w r3, [r4], #24\n", - " a: f854 3c18 ldr.w r3, [r4, #-24]\n", - " e: f854 3d18 ldr.w r3, [r4, #-24]!\n", - " 12: f854 3918 ldr.w r3, [r4], #-24\n", - " 16: 7e23 ldrb r3, [r4, #24]\n", - " 18: f814 3f18 ldrb.w r3, [r4, #24]!\n", - " 1c: f814 3b18 ldrb.w r3, [r4], #24\n", - " 20: f814 3c18 ldrb.w r3, [r4, #-24]\n", - " 24: f814 3d18 ldrb.w r3, [r4, #-24]!\n", - " 28: f814 3918 ldrb.w r3, [r4], #-24\n", - " 2c: 8b23 ldrh r3, [r4, #24]\n", - " 2e: f834 3f18 ldrh.w r3, [r4, #24]!\n", - " 32: f834 3b18 ldrh.w r3, [r4], #24\n", - " 36: f834 3c18 ldrh.w r3, [r4, #-24]\n", - " 3a: f834 3d18 ldrh.w r3, [r4, #-24]!\n", - " 3e: f834 3918 ldrh.w r3, [r4], #-24\n", - " 42: f994 3018 ldrsb.w r3, [r4, #24]\n", - " 46: f914 3f18 ldrsb.w r3, [r4, #24]!\n", - " 4a: f914 3b18 ldrsb.w r3, [r4], #24\n", - " 4e: f914 3c18 ldrsb.w r3, [r4, #-24]\n", - " 52: f914 3d18 ldrsb.w r3, [r4, #-24]!\n", - " 56: f914 3918 ldrsb.w r3, [r4], #-24\n", - " 5a: f9b4 3018 ldrsh.w r3, [r4, #24]\n", - " 5e: f934 3f18 ldrsh.w r3, [r4, #24]!\n", - " 62: f934 3b18 ldrsh.w r3, [r4], #24\n", - " 66: f934 3c18 ldrsh.w r3, [r4, #-24]\n", - " 6a: f934 3d18 ldrsh.w r3, [r4, #-24]!\n", - " 6e: f934 3918 ldrsh.w r3, [r4], #-24\n", - nullptr -}; -const char* const ComplexStoreResults[] = { - " 0: 61a3 str r3, [r4, #24]\n", - " 2: f844 3f18 str.w r3, [r4, #24]!\n", - " 6: f844 3b18 str.w r3, [r4], #24\n", - " a: f844 3c18 str.w r3, [r4, #-24]\n", - " e: f844 3d18 str.w r3, [r4, #-24]!\n", - " 12: f844 3918 str.w r3, [r4], #-24\n", - " 16: 7623 strb r3, [r4, #24]\n", - " 18: f804 3f18 strb.w r3, [r4, #24]!\n", - " 1c: f804 3b18 strb.w r3, [r4], #24\n", - " 20: f804 3c18 strb.w r3, [r4, #-24]\n", - " 24: f804 3d18 strb.w r3, [r4, #-24]!\n", - " 28: f804 3918 strb.w r3, [r4], #-24\n", - " 2c: 8323 strh r3, [r4, #24]\n", - " 2e: f824 3f18 strh.w r3, [r4, #24]!\n", - " 32: f824 3b18 strh.w r3, [r4], #24\n", - " 36: f824 3c18 strh.w r3, [r4, #-24]\n", - " 3a: f824 3d18 strh.w r3, [r4, #-24]!\n", - " 3e: f824 3918 strh.w r3, [r4], #-24\n", - nullptr -}; -const char* const NegativeLoadStoreResults[] = { - " 0: f854 3c18 ldr.w r3, [r4, #-24]\n", - " 4: f854 3d18 ldr.w r3, [r4, #-24]!\n", - " 8: f854 3918 ldr.w r3, [r4], #-24\n", - " c: f854 3e18 ldrt r3, [r4, #24]\n", - " 10: f854 3f18 ldr.w r3, [r4, #24]!\n", - " 14: f854 3b18 ldr.w r3, [r4], #24\n", - " 18: f814 3c18 ldrb.w r3, [r4, #-24]\n", - " 1c: f814 3d18 ldrb.w r3, [r4, #-24]!\n", - " 20: f814 3918 ldrb.w r3, [r4], #-24\n", - " 24: f814 3e18 ldrbt r3, [r4, #24]\n", - " 28: f814 3f18 ldrb.w r3, [r4, #24]!\n", - " 2c: f814 3b18 ldrb.w r3, [r4], #24\n", - " 30: f834 3c18 ldrh.w r3, [r4, #-24]\n", - " 34: f834 3d18 ldrh.w r3, [r4, #-24]!\n", - " 38: f834 3918 ldrh.w r3, [r4], #-24\n", - " 3c: f834 3e18 ldrht r3, [r4, #24]\n", - " 40: f834 3f18 ldrh.w r3, [r4, #24]!\n", - " 44: f834 3b18 ldrh.w r3, [r4], #24\n", - " 48: f914 3c18 ldrsb.w r3, [r4, #-24]\n", - " 4c: f914 3d18 ldrsb.w r3, [r4, #-24]!\n", - " 50: f914 3918 ldrsb.w r3, [r4], #-24\n", - " 54: f914 3e18 ldrsbt r3, [r4, #24]\n", - " 58: f914 3f18 ldrsb.w r3, [r4, #24]!\n", - " 5c: f914 3b18 ldrsb.w r3, [r4], #24\n", - " 60: f934 3c18 ldrsh.w r3, [r4, #-24]\n", - " 64: f934 3d18 ldrsh.w r3, [r4, #-24]!\n", - " 68: f934 3918 ldrsh.w r3, [r4], #-24\n", - " 6c: f934 3e18 ldrsht r3, [r4, #24]\n", - " 70: f934 3f18 ldrsh.w r3, [r4, #24]!\n", - " 74: f934 3b18 ldrsh.w r3, [r4], #24\n", - " 78: f844 3c18 str.w r3, [r4, #-24]\n", - " 7c: f844 3d18 str.w r3, [r4, #-24]!\n", - " 80: f844 3918 str.w r3, [r4], #-24\n", - " 84: f844 3e18 strt r3, [r4, #24]\n", - " 88: f844 3f18 str.w r3, [r4, #24]!\n", - " 8c: f844 3b18 str.w r3, [r4], #24\n", - " 90: f804 3c18 strb.w r3, [r4, #-24]\n", - " 94: f804 3d18 strb.w r3, [r4, #-24]!\n", - " 98: f804 3918 strb.w r3, [r4], #-24\n", - " 9c: f804 3e18 strbt r3, [r4, #24]\n", - " a0: f804 3f18 strb.w r3, [r4, #24]!\n", - " a4: f804 3b18 strb.w r3, [r4], #24\n", - " a8: f824 3c18 strh.w r3, [r4, #-24]\n", - " ac: f824 3d18 strh.w r3, [r4, #-24]!\n", - " b0: f824 3918 strh.w r3, [r4], #-24\n", - " b4: f824 3e18 strht r3, [r4, #24]\n", - " b8: f824 3f18 strh.w r3, [r4, #24]!\n", - " bc: f824 3b18 strh.w r3, [r4], #24\n", - nullptr -}; -const char* const SimpleLoadStoreDualResults[] = { - " 0: e9c0 2306 strd r2, r3, [r0, #24]\n", - " 4: e9d0 2306 ldrd r2, r3, [r0, #24]\n", - nullptr -}; -const char* const ComplexLoadStoreDualResults[] = { - " 0: e9c0 2306 strd r2, r3, [r0, #24]\n", - " 4: e9e0 2306 strd r2, r3, [r0, #24]!\n", - " 8: e8e0 2306 strd r2, r3, [r0], #24\n", - " c: e940 2306 strd r2, r3, [r0, #-24]\n", - " 10: e960 2306 strd r2, r3, [r0, #-24]!\n", - " 14: e860 2306 strd r2, r3, [r0], #-24\n", - " 18: e9d0 2306 ldrd r2, r3, [r0, #24]\n", - " 1c: e9f0 2306 ldrd r2, r3, [r0, #24]!\n", - " 20: e8f0 2306 ldrd r2, r3, [r0], #24\n", - " 24: e950 2306 ldrd r2, r3, [r0, #-24]\n", - " 28: e970 2306 ldrd r2, r3, [r0, #-24]!\n", - " 2c: e870 2306 ldrd r2, r3, [r0], #-24\n", - nullptr -}; -const char* const NegativeLoadStoreDualResults[] = { - " 0: e940 2306 strd r2, r3, [r0, #-24]\n", - " 4: e960 2306 strd r2, r3, [r0, #-24]!\n", - " 8: e860 2306 strd r2, r3, [r0], #-24\n", - " c: e9c0 2306 strd r2, r3, [r0, #24]\n", - " 10: e9e0 2306 strd r2, r3, [r0, #24]!\n", - " 14: e8e0 2306 strd r2, r3, [r0], #24\n", - " 18: e950 2306 ldrd r2, r3, [r0, #-24]\n", - " 1c: e970 2306 ldrd r2, r3, [r0, #-24]!\n", - " 20: e870 2306 ldrd r2, r3, [r0], #-24\n", - " 24: e9d0 2306 ldrd r2, r3, [r0, #24]\n", - " 28: e9f0 2306 ldrd r2, r3, [r0, #24]!\n", - " 2c: e8f0 2306 ldrd r2, r3, [r0], #24\n", - nullptr -}; -const char* const SimpleBranchResults[] = { - " 0: 2002 movs r0, #2\n", - " 2: 2101 movs r1, #1\n", - " 4: e7fd b.n 2 <SimpleBranch+0x2>\n", - " 6: e000 b.n a <SimpleBranch+0xa>\n", - " 8: 2102 movs r1, #2\n", - " a: 2003 movs r0, #3\n", - " c: 2002 movs r0, #2\n", - " e: 2101 movs r1, #1\n", - " 10: d0fd beq.n e <SimpleBranch+0xe>\n", - " 12: d000 beq.n 16 <SimpleBranch+0x16>\n", - " 14: 2102 movs r1, #2\n", - " 16: 2003 movs r0, #3\n", - " 18: e002 b.n 20 <SimpleBranch+0x20>\n", - " 1a: 2104 movs r1, #4\n", - " 1c: e000 b.n 20 <SimpleBranch+0x20>\n", - " 1e: 2105 movs r1, #5\n", - " 20: 2006 movs r0, #6\n", - nullptr -}; -const char* const LongBranchResults[] = { - " 0: f04f 0002 mov.w r0, #2\n", - " 4: f04f 0101 mov.w r1, #1\n", - " 8: f7ff bffc b.w 4 <LongBranch+0x4>\n", - " c: f000 b802 b.w 14 <LongBranch+0x14>\n", - " 10: f04f 0102 mov.w r1, #2\n", - " 14: f04f 0003 mov.w r0, #3\n", - " 18: f04f 0002 mov.w r0, #2\n", - " 1c: f04f 0101 mov.w r1, #1\n", - " 20: f43f affc beq.w 1c <LongBranch+0x1c>\n", - " 24: f000 8002 beq.w 2c <LongBranch+0x2c>\n", - " 28: f04f 0102 mov.w r1, #2\n", - " 2c: f04f 0003 mov.w r0, #3\n", - " 30: f000 b806 b.w 40 <LongBranch+0x40>\n", - " 34: f04f 0104 mov.w r1, #4\n", - " 38: f000 b802 b.w 40 <LongBranch+0x40>\n", - " 3c: f04f 0105 mov.w r1, #5\n", - " 40: f04f 0006 mov.w r0, #6\n", - nullptr -}; -const char* const LoadMultipleResults[] = { - " 0: cc09 ldmia r4!, {r0, r3}\n", - " 2: e934 4800 ldmdb r4!, {fp, lr}\n", - " 6: e914 4800 ldmdb r4, {fp, lr}\n", - " a: f854 5b04 ldr.w r5, [r4], #4\n", - nullptr -}; -const char* const StoreMultipleResults[] = { - " 0: c409 stmia r4!, {r0, r3}\n", - " 2: e8a4 4800 stmia.w r4!, {fp, lr}\n", - " 6: e884 4800 stmia.w r4, {fp, lr}\n", - " a: f844 5c04 str.w r5, [r4, #-4]\n", - " e: f844 5d04 str.w r5, [r4, #-4]!\n", - nullptr -}; -const char* const MovWMovTResults[] = { - " 0: f240 0400 movw r4, #0\n", - " 4: f240 0434 movw r4, #52 ; 0x34\n", - " 8: f240 0934 movw r9, #52 ; 0x34\n", - " c: f241 2334 movw r3, #4660 ; 0x1234\n", - " 10: f64f 79ff movw r9, #65535 ; 0xffff\n", - " 14: f2c0 0000 movt r0, #0\n", - " 18: f2c1 2034 movt r0, #4660 ; 0x1234\n", - " 1c: f6cf 71ff movt r1, #65535 ; 0xffff\n", - nullptr -}; -const char* const SpecialAddSubResults[] = { - " 0: aa14 add r2, sp, #80 ; 0x50\n", - " 2: b014 add sp, #80 ; 0x50\n", - " 4: f10d 0850 add.w r8, sp, #80 ; 0x50\n", - " 8: f50d 6270 add.w r2, sp, #3840 ; 0xf00\n", - " c: f50d 6d70 add.w sp, sp, #3840 ; 0xf00\n", - " 10: f60d 7dfc addw sp, sp, #4092 ; 0xffc\n", - " 14: b094 sub sp, #80 ; 0x50\n", - " 16: f1ad 0050 sub.w r0, sp, #80 ; 0x50\n", - " 1a: f1ad 0850 sub.w r8, sp, #80 ; 0x50\n", - " 1e: f5ad 6d70 sub.w sp, sp, #3840 ; 0xf00\n", - " 22: f6ad 7dfc subw sp, sp, #4092 ; 0xffc\n", - nullptr -}; -const char* const LoadFromOffsetResults[] = { - " 0: 68e2 ldr r2, [r4, #12]\n", - " 2: f8d4 2fff ldr.w r2, [r4, #4095] ; 0xfff\n", - " 6: f504 5280 add.w r2, r4, #4096 ; 0x1000\n", - " a: 6812 ldr r2, [r2, #0]\n", - " c: f504 1280 add.w r2, r4, #1048576 ; 0x100000\n", - " 10: f8d2 20a4 ldr.w r2, [r2, #164] ; 0xa4\n", - " 14: f241 0200 movw r2, #4096 ; 0x1000\n", - " 18: f2c0 0210 movt r2, #16\n", - " 1c: 4422 add r2, r4\n", - " 1e: 6812 ldr r2, [r2, #0]\n", - " 20: f241 0c00 movw ip, #4096 ; 0x1000\n", - " 24: f2c0 0c10 movt ip, #16\n", - " 28: 4464 add r4, ip\n", - " 2a: 6824 ldr r4, [r4, #0]\n", - " 2c: 89a2 ldrh r2, [r4, #12]\n", - " 2e: f8b4 2fff ldrh.w r2, [r4, #4095] ; 0xfff\n", - " 32: f504 5280 add.w r2, r4, #4096 ; 0x1000\n", - " 36: 8812 ldrh r2, [r2, #0]\n", - " 38: f504 1280 add.w r2, r4, #1048576 ; 0x100000\n", - " 3c: f8b2 20a4 ldrh.w r2, [r2, #164] ; 0xa4\n", - " 40: f241 0200 movw r2, #4096 ; 0x1000\n", - " 44: f2c0 0210 movt r2, #16\n", - " 48: 4422 add r2, r4\n", - " 4a: 8812 ldrh r2, [r2, #0]\n", - " 4c: f241 0c00 movw ip, #4096 ; 0x1000\n", - " 50: f2c0 0c10 movt ip, #16\n", - " 54: 4464 add r4, ip\n", - " 56: 8824 ldrh r4, [r4, #0]\n", - " 58: e9d4 2303 ldrd r2, r3, [r4, #12]\n", - " 5c: e9d4 23ff ldrd r2, r3, [r4, #1020] ; 0x3fc\n", - " 60: f504 6280 add.w r2, r4, #1024 ; 0x400\n", - " 64: e9d2 2300 ldrd r2, r3, [r2]\n", - " 68: f504 2280 add.w r2, r4, #262144 ; 0x40000\n", - " 6c: e9d2 2329 ldrd r2, r3, [r2, #164]; 0xa4\n", - " 70: f240 4200 movw r2, #1024 ; 0x400\n", - " 74: f2c0 0204 movt r2, #4\n", - " 78: 4422 add r2, r4\n", - " 7a: e9d2 2300 ldrd r2, r3, [r2]\n", - " 7e: f240 4c00 movw ip, #1024 ; 0x400\n", - " 82: f2c0 0c04 movt ip, #4\n", - " 86: 4464 add r4, ip\n", - " 88: e9d4 4500 ldrd r4, r5, [r4]\n", - " 8c: f8dc 000c ldr.w r0, [ip, #12]\n", - " 90: f5a4 1280 sub.w r2, r4, #1048576 ; 0x100000\n", - " 94: f8d2 20a4 ldr.w r2, [r2, #164] ; 0xa4\n", - " 98: f994 200c ldrsb.w r2, [r4, #12]\n", - " 9c: 7b22 ldrb r2, [r4, #12]\n", - " 9e: f9b4 200c ldrsh.w r2, [r4, #12]\n", - nullptr -}; -const char* const StoreToOffsetResults[] = { - " 0: 60e2 str r2, [r4, #12]\n", - " 2: f8c4 2fff str.w r2, [r4, #4095] ; 0xfff\n", - " 6: f504 5c80 add.w ip, r4, #4096 ; 0x1000\n", - " a: f8cc 2000 str.w r2, [ip]\n", - " e: f504 1c80 add.w ip, r4, #1048576 ; 0x100000\n", - " 12: f8cc 20a4 str.w r2, [ip, #164] ; 0xa4\n", - " 16: f241 0c00 movw ip, #4096 ; 0x1000\n", - " 1a: f2c0 0c10 movt ip, #16\n", - " 1e: 44a4 add ip, r4\n", - " 20: f8cc 2000 str.w r2, [ip]\n", - " 24: f241 0c00 movw ip, #4096 ; 0x1000\n", - " 28: f2c0 0c10 movt ip, #16\n", - " 2c: 44a4 add ip, r4\n", - " 2e: f8cc 4000 str.w r4, [ip]\n", - " 32: 81a2 strh r2, [r4, #12]\n", - " 34: f8a4 2fff strh.w r2, [r4, #4095] ; 0xfff\n", - " 38: f504 5c80 add.w ip, r4, #4096 ; 0x1000\n", - " 3c: f8ac 2000 strh.w r2, [ip]\n", - " 40: f504 1c80 add.w ip, r4, #1048576 ; 0x100000\n", - " 44: f8ac 20a4 strh.w r2, [ip, #164] ; 0xa4\n", - " 48: f241 0c00 movw ip, #4096 ; 0x1000\n", - " 4c: f2c0 0c10 movt ip, #16\n", - " 50: 44a4 add ip, r4\n", - " 52: f8ac 2000 strh.w r2, [ip]\n", - " 56: f241 0c00 movw ip, #4096 ; 0x1000\n", - " 5a: f2c0 0c10 movt ip, #16\n", - " 5e: 44a4 add ip, r4\n", - " 60: f8ac 4000 strh.w r4, [ip]\n", - " 64: e9c4 2303 strd r2, r3, [r4, #12]\n", - " 68: e9c4 23ff strd r2, r3, [r4, #1020] ; 0x3fc\n", - " 6c: f504 6c80 add.w ip, r4, #1024 ; 0x400\n", - " 70: e9cc 2300 strd r2, r3, [ip]\n", - " 74: f504 2c80 add.w ip, r4, #262144 ; 0x40000\n", - " 78: e9cc 2329 strd r2, r3, [ip, #164]; 0xa4\n", - " 7c: f240 4c00 movw ip, #1024 ; 0x400\n", - " 80: f2c0 0c04 movt ip, #4\n", - " 84: 44a4 add ip, r4\n", - " 86: e9cc 2300 strd r2, r3, [ip]\n", - " 8a: f240 4c00 movw ip, #1024 ; 0x400\n", - " 8e: f2c0 0c04 movt ip, #4\n", - " 92: 44a4 add ip, r4\n", - " 94: e9cc 4500 strd r4, r5, [ip]\n", - " 98: f8cc 000c str.w r0, [ip, #12]\n", - " 9c: f5a4 1c80 sub.w ip, r4, #1048576 ; 0x100000\n", - " a0: f8cc 20a4 str.w r2, [ip, #164] ; 0xa4\n", - " a4: 7322 strb r2, [r4, #12]\n", - nullptr -}; -const char* const IfThenResults[] = { - " 0: bf08 it eq\n", - " 2: 2101 moveq r1, #1\n", - " 4: bf04 itt eq\n", - " 6: 2101 moveq r1, #1\n", - " 8: 2202 moveq r2, #2\n", - " a: bf0c ite eq\n", - " c: 2101 moveq r1, #1\n", - " e: 2202 movne r2, #2\n", - " 10: bf06 itte eq\n", - " 12: 2101 moveq r1, #1\n", - " 14: 2202 moveq r2, #2\n", - " 16: 2303 movne r3, #3\n", - " 18: bf0e itee eq\n", - " 1a: 2101 moveq r1, #1\n", - " 1c: 2202 movne r2, #2\n", - " 1e: 2303 movne r3, #3\n", - " 20: bf03 ittte eq\n", - " 22: 2101 moveq r1, #1\n", - " 24: 2202 moveq r2, #2\n", - " 26: 2303 moveq r3, #3\n", - " 28: 2404 movne r4, #4\n", - nullptr -}; -const char* const CbzCbnzResults[] = { - " 0: b10a cbz r2, 6 <CbzCbnz+0x6>\n", - " 2: 2103 movs r1, #3\n", - " 4: 2203 movs r2, #3\n", - " 6: 2204 movs r2, #4\n", - " 8: b912 cbnz r2, 10 <CbzCbnz+0x10>\n", - " a: f04f 0803 mov.w r8, #3\n", - " e: 2203 movs r2, #3\n", - " 10: 2204 movs r2, #4\n", - nullptr -}; -const char* const MultiplyResults[] = { - " 0: 4348 muls r0, r1\n", - " 2: fb01 f002 mul.w r0, r1, r2\n", - " 6: fb09 f808 mul.w r8, r9, r8\n", - " a: fb09 f80a mul.w r8, r9, sl\n", - " e: fb01 3002 mla r0, r1, r2, r3\n", - " 12: fb09 9808 mla r8, r9, r8, r9\n", - " 16: fb01 3012 mls r0, r1, r2, r3\n", - " 1a: fb09 9818 mls r8, r9, r8, r9\n", - " 1e: fba2 0103 umull r0, r1, r2, r3\n", - " 22: fbaa 890b umull r8, r9, sl, fp\n", - nullptr -}; -const char* const DivideResults[] = { - " 0: fb91 f0f2 sdiv r0, r1, r2\n", - " 4: fb99 f8fa sdiv r8, r9, sl\n", - " 8: fbb1 f0f2 udiv r0, r1, r2\n", - " c: fbb9 f8fa udiv r8, r9, sl\n", - nullptr -}; -const char* const VMovResults[] = { - " 0: eef7 0a00 vmov.f32 s1, #112 ; 0x70\n", - " 4: eeb7 1b00 vmov.f64 d1, #112 ; 0x70\n", - " 8: eef0 0a41 vmov.f32 s1, s2\n", - " c: eeb0 1b42 vmov.f64 d1, d2\n", - nullptr -}; -const char* const BasicFloatingPointResults[] = { - " 0: ee30 0a81 vadd.f32 s0, s1, s2\n", - " 4: ee30 0ac1 vsub.f32 s0, s1, s2\n", - " 8: ee20 0a81 vmul.f32 s0, s1, s2\n", - " c: ee00 0a81 vmla.f32 s0, s1, s2\n", - " 10: ee00 0ac1 vmls.f32 s0, s1, s2\n", - " 14: ee80 0a81 vdiv.f32 s0, s1, s2\n", - " 18: eeb0 0ae0 vabs.f32 s0, s1\n", - " 1c: eeb1 0a60 vneg.f32 s0, s1\n", - " 20: eeb1 0ae0 vsqrt.f32 s0, s1\n", - " 24: ee31 0b02 vadd.f64 d0, d1, d2\n", - " 28: ee31 0b42 vsub.f64 d0, d1, d2\n", - " 2c: ee21 0b02 vmul.f64 d0, d1, d2\n", - " 30: ee01 0b02 vmla.f64 d0, d1, d2\n", - " 34: ee01 0b42 vmls.f64 d0, d1, d2\n", - " 38: ee81 0b02 vdiv.f64 d0, d1, d2\n", - " 3c: eeb0 0bc1 vabs.f64 d0, d1\n", - " 40: eeb1 0b41 vneg.f64 d0, d1\n", - " 44: eeb1 0bc1 vsqrt.f64 d0, d1\n", - nullptr -}; -const char* const FloatingPointConversionsResults[] = { - " 0: eeb7 1bc2 vcvt.f32.f64 s2, d2\n", - " 4: eeb7 2ac1 vcvt.f64.f32 d2, s2\n", - " 8: eefd 0ac1 vcvt.s32.f32 s1, s2\n", - " c: eef8 0ac1 vcvt.f32.s32 s1, s2\n", - " 10: eefd 0bc2 vcvt.s32.f64 s1, d2\n", - " 14: eeb8 1bc1 vcvt.f64.s32 d1, s2\n", - " 18: eefc 0ac1 vcvt.u32.f32 s1, s2\n", - " 1c: eef8 0a41 vcvt.f32.u32 s1, s2\n", - " 20: eefc 0bc2 vcvt.u32.f64 s1, d2\n", - " 24: eeb8 1b41 vcvt.f64.u32 d1, s2\n", - nullptr -}; -const char* const FloatingPointComparisonsResults[] = { - " 0: eeb4 0a60 vcmp.f32 s0, s1\n", - " 4: eeb4 0b41 vcmp.f64 d0, d1\n", - " 8: eeb5 1a40 vcmp.f32 s2, #0.0\n", - " c: eeb5 2b40 vcmp.f64 d2, #0.0\n", - nullptr -}; -const char* const CallsResults[] = { - " 0: 47f0 blx lr\n", - " 2: 4770 bx lr\n", - nullptr -}; -const char* const BreakpointResults[] = { - " 0: be00 bkpt 0x0000\n", - nullptr -}; -const char* const StrR1Results[] = { - " 0: 9111 str r1, [sp, #68] ; 0x44\n", - " 2: f8cd 142c str.w r1, [sp, #1068] ; 0x42c\n", - nullptr -}; -const char* const VPushPopResults[] = { - " 0: ed2d 1a04 vpush {s2-s5}\n", - " 4: ed2d 2b08 vpush {d2-d5}\n", - " 8: ecbd 1a04 vpop {s2-s5}\n", - " c: ecbd 2b08 vpop {d2-d5}\n", - nullptr -}; -const char* const Max16BitBranchResults[] = { - " 0: e3ff b.n 802 <Max16BitBranch+0x802>\n", - " 2: 2300 movs r3, #0\n", - " 4: 2302 movs r3, #2\n", - " 6: 2304 movs r3, #4\n", - " 8: 2306 movs r3, #6\n", - " a: 2308 movs r3, #8\n", - " c: 230a movs r3, #10\n", - " e: 230c movs r3, #12\n", - " 10: 230e movs r3, #14\n", - " 12: 2310 movs r3, #16\n", - " 14: 2312 movs r3, #18\n", - " 16: 2314 movs r3, #20\n", - " 18: 2316 movs r3, #22\n", - " 1a: 2318 movs r3, #24\n", - " 1c: 231a movs r3, #26\n", - " 1e: 231c movs r3, #28\n", - " 20: 231e movs r3, #30\n", - " 22: 2320 movs r3, #32\n", - " 24: 2322 movs r3, #34 ; 0x22\n", - " 26: 2324 movs r3, #36 ; 0x24\n", - " 28: 2326 movs r3, #38 ; 0x26\n", - " 2a: 2328 movs r3, #40 ; 0x28\n", - " 2c: 232a movs r3, #42 ; 0x2a\n", - " 2e: 232c movs r3, #44 ; 0x2c\n", - " 30: 232e movs r3, #46 ; 0x2e\n", - " 32: 2330 movs r3, #48 ; 0x30\n", - " 34: 2332 movs r3, #50 ; 0x32\n", - " 36: 2334 movs r3, #52 ; 0x34\n", - " 38: 2336 movs r3, #54 ; 0x36\n", - " 3a: 2338 movs r3, #56 ; 0x38\n", - " 3c: 233a movs r3, #58 ; 0x3a\n", - " 3e: 233c movs r3, #60 ; 0x3c\n", - " 40: 233e movs r3, #62 ; 0x3e\n", - " 42: 2340 movs r3, #64 ; 0x40\n", - " 44: 2342 movs r3, #66 ; 0x42\n", - " 46: 2344 movs r3, #68 ; 0x44\n", - " 48: 2346 movs r3, #70 ; 0x46\n", - " 4a: 2348 movs r3, #72 ; 0x48\n", - " 4c: 234a movs r3, #74 ; 0x4a\n", - " 4e: 234c movs r3, #76 ; 0x4c\n", - " 50: 234e movs r3, #78 ; 0x4e\n", - " 52: 2350 movs r3, #80 ; 0x50\n", - " 54: 2352 movs r3, #82 ; 0x52\n", - " 56: 2354 movs r3, #84 ; 0x54\n", - " 58: 2356 movs r3, #86 ; 0x56\n", - " 5a: 2358 movs r3, #88 ; 0x58\n", - " 5c: 235a movs r3, #90 ; 0x5a\n", - " 5e: 235c movs r3, #92 ; 0x5c\n", - " 60: 235e movs r3, #94 ; 0x5e\n", - " 62: 2360 movs r3, #96 ; 0x60\n", - " 64: 2362 movs r3, #98 ; 0x62\n", - " 66: 2364 movs r3, #100 ; 0x64\n", - " 68: 2366 movs r3, #102 ; 0x66\n", - " 6a: 2368 movs r3, #104 ; 0x68\n", - " 6c: 236a movs r3, #106 ; 0x6a\n", - " 6e: 236c movs r3, #108 ; 0x6c\n", - " 70: 236e movs r3, #110 ; 0x6e\n", - " 72: 2370 movs r3, #112 ; 0x70\n", - " 74: 2372 movs r3, #114 ; 0x72\n", - " 76: 2374 movs r3, #116 ; 0x74\n", - " 78: 2376 movs r3, #118 ; 0x76\n", - " 7a: 2378 movs r3, #120 ; 0x78\n", - " 7c: 237a movs r3, #122 ; 0x7a\n", - " 7e: 237c movs r3, #124 ; 0x7c\n", - " 80: 237e movs r3, #126 ; 0x7e\n", - " 82: 2380 movs r3, #128 ; 0x80\n", - " 84: 2382 movs r3, #130 ; 0x82\n", - " 86: 2384 movs r3, #132 ; 0x84\n", - " 88: 2386 movs r3, #134 ; 0x86\n", - " 8a: 2388 movs r3, #136 ; 0x88\n", - " 8c: 238a movs r3, #138 ; 0x8a\n", - " 8e: 238c movs r3, #140 ; 0x8c\n", - " 90: 238e movs r3, #142 ; 0x8e\n", - " 92: 2390 movs r3, #144 ; 0x90\n", - " 94: 2392 movs r3, #146 ; 0x92\n", - " 96: 2394 movs r3, #148 ; 0x94\n", - " 98: 2396 movs r3, #150 ; 0x96\n", - " 9a: 2398 movs r3, #152 ; 0x98\n", - " 9c: 239a movs r3, #154 ; 0x9a\n", - " 9e: 239c movs r3, #156 ; 0x9c\n", - " a0: 239e movs r3, #158 ; 0x9e\n", - " a2: 23a0 movs r3, #160 ; 0xa0\n", - " a4: 23a2 movs r3, #162 ; 0xa2\n", - " a6: 23a4 movs r3, #164 ; 0xa4\n", - " a8: 23a6 movs r3, #166 ; 0xa6\n", - " aa: 23a8 movs r3, #168 ; 0xa8\n", - " ac: 23aa movs r3, #170 ; 0xaa\n", - " ae: 23ac movs r3, #172 ; 0xac\n", - " b0: 23ae movs r3, #174 ; 0xae\n", - " b2: 23b0 movs r3, #176 ; 0xb0\n", - " b4: 23b2 movs r3, #178 ; 0xb2\n", - " b6: 23b4 movs r3, #180 ; 0xb4\n", - " b8: 23b6 movs r3, #182 ; 0xb6\n", - " ba: 23b8 movs r3, #184 ; 0xb8\n", - " bc: 23ba movs r3, #186 ; 0xba\n", - " be: 23bc movs r3, #188 ; 0xbc\n", - " c0: 23be movs r3, #190 ; 0xbe\n", - " c2: 23c0 movs r3, #192 ; 0xc0\n", - " c4: 23c2 movs r3, #194 ; 0xc2\n", - " c6: 23c4 movs r3, #196 ; 0xc4\n", - " c8: 23c6 movs r3, #198 ; 0xc6\n", - " ca: 23c8 movs r3, #200 ; 0xc8\n", - " cc: 23ca movs r3, #202 ; 0xca\n", - " ce: 23cc movs r3, #204 ; 0xcc\n", - " d0: 23ce movs r3, #206 ; 0xce\n", - " d2: 23d0 movs r3, #208 ; 0xd0\n", - " d4: 23d2 movs r3, #210 ; 0xd2\n", - " d6: 23d4 movs r3, #212 ; 0xd4\n", - " d8: 23d6 movs r3, #214 ; 0xd6\n", - " da: 23d8 movs r3, #216 ; 0xd8\n", - " dc: 23da movs r3, #218 ; 0xda\n", - " de: 23dc movs r3, #220 ; 0xdc\n", - " e0: 23de movs r3, #222 ; 0xde\n", - " e2: 23e0 movs r3, #224 ; 0xe0\n", - " e4: 23e2 movs r3, #226 ; 0xe2\n", - " e6: 23e4 movs r3, #228 ; 0xe4\n", - " e8: 23e6 movs r3, #230 ; 0xe6\n", - " ea: 23e8 movs r3, #232 ; 0xe8\n", - " ec: 23ea movs r3, #234 ; 0xea\n", - " ee: 23ec movs r3, #236 ; 0xec\n", - " f0: 23ee movs r3, #238 ; 0xee\n", - " f2: 23f0 movs r3, #240 ; 0xf0\n", - " f4: 23f2 movs r3, #242 ; 0xf2\n", - " f6: 23f4 movs r3, #244 ; 0xf4\n", - " f8: 23f6 movs r3, #246 ; 0xf6\n", - " fa: 23f8 movs r3, #248 ; 0xf8\n", - " fc: 23fa movs r3, #250 ; 0xfa\n", - " fe: 23fc movs r3, #252 ; 0xfc\n", - " 100: 23fe movs r3, #254 ; 0xfe\n", - " 102: 2300 movs r3, #0\n", - " 104: 2302 movs r3, #2\n", - " 106: 2304 movs r3, #4\n", - " 108: 2306 movs r3, #6\n", - " 10a: 2308 movs r3, #8\n", - " 10c: 230a movs r3, #10\n", - " 10e: 230c movs r3, #12\n", - " 110: 230e movs r3, #14\n", - " 112: 2310 movs r3, #16\n", - " 114: 2312 movs r3, #18\n", - " 116: 2314 movs r3, #20\n", - " 118: 2316 movs r3, #22\n", - " 11a: 2318 movs r3, #24\n", - " 11c: 231a movs r3, #26\n", - " 11e: 231c movs r3, #28\n", - " 120: 231e movs r3, #30\n", - " 122: 2320 movs r3, #32\n", - " 124: 2322 movs r3, #34 ; 0x22\n", - " 126: 2324 movs r3, #36 ; 0x24\n", - " 128: 2326 movs r3, #38 ; 0x26\n", - " 12a: 2328 movs r3, #40 ; 0x28\n", - " 12c: 232a movs r3, #42 ; 0x2a\n", - " 12e: 232c movs r3, #44 ; 0x2c\n", - " 130: 232e movs r3, #46 ; 0x2e\n", - " 132: 2330 movs r3, #48 ; 0x30\n", - " 134: 2332 movs r3, #50 ; 0x32\n", - " 136: 2334 movs r3, #52 ; 0x34\n", - " 138: 2336 movs r3, #54 ; 0x36\n", - " 13a: 2338 movs r3, #56 ; 0x38\n", - " 13c: 233a movs r3, #58 ; 0x3a\n", - " 13e: 233c movs r3, #60 ; 0x3c\n", - " 140: 233e movs r3, #62 ; 0x3e\n", - " 142: 2340 movs r3, #64 ; 0x40\n", - " 144: 2342 movs r3, #66 ; 0x42\n", - " 146: 2344 movs r3, #68 ; 0x44\n", - " 148: 2346 movs r3, #70 ; 0x46\n", - " 14a: 2348 movs r3, #72 ; 0x48\n", - " 14c: 234a movs r3, #74 ; 0x4a\n", - " 14e: 234c movs r3, #76 ; 0x4c\n", - " 150: 234e movs r3, #78 ; 0x4e\n", - " 152: 2350 movs r3, #80 ; 0x50\n", - " 154: 2352 movs r3, #82 ; 0x52\n", - " 156: 2354 movs r3, #84 ; 0x54\n", - " 158: 2356 movs r3, #86 ; 0x56\n", - " 15a: 2358 movs r3, #88 ; 0x58\n", - " 15c: 235a movs r3, #90 ; 0x5a\n", - " 15e: 235c movs r3, #92 ; 0x5c\n", - " 160: 235e movs r3, #94 ; 0x5e\n", - " 162: 2360 movs r3, #96 ; 0x60\n", - " 164: 2362 movs r3, #98 ; 0x62\n", - " 166: 2364 movs r3, #100 ; 0x64\n", - " 168: 2366 movs r3, #102 ; 0x66\n", - " 16a: 2368 movs r3, #104 ; 0x68\n", - " 16c: 236a movs r3, #106 ; 0x6a\n", - " 16e: 236c movs r3, #108 ; 0x6c\n", - " 170: 236e movs r3, #110 ; 0x6e\n", - " 172: 2370 movs r3, #112 ; 0x70\n", - " 174: 2372 movs r3, #114 ; 0x72\n", - " 176: 2374 movs r3, #116 ; 0x74\n", - " 178: 2376 movs r3, #118 ; 0x76\n", - " 17a: 2378 movs r3, #120 ; 0x78\n", - " 17c: 237a movs r3, #122 ; 0x7a\n", - " 17e: 237c movs r3, #124 ; 0x7c\n", - " 180: 237e movs r3, #126 ; 0x7e\n", - " 182: 2380 movs r3, #128 ; 0x80\n", - " 184: 2382 movs r3, #130 ; 0x82\n", - " 186: 2384 movs r3, #132 ; 0x84\n", - " 188: 2386 movs r3, #134 ; 0x86\n", - " 18a: 2388 movs r3, #136 ; 0x88\n", - " 18c: 238a movs r3, #138 ; 0x8a\n", - " 18e: 238c movs r3, #140 ; 0x8c\n", - " 190: 238e movs r3, #142 ; 0x8e\n", - " 192: 2390 movs r3, #144 ; 0x90\n", - " 194: 2392 movs r3, #146 ; 0x92\n", - " 196: 2394 movs r3, #148 ; 0x94\n", - " 198: 2396 movs r3, #150 ; 0x96\n", - " 19a: 2398 movs r3, #152 ; 0x98\n", - " 19c: 239a movs r3, #154 ; 0x9a\n", - " 19e: 239c movs r3, #156 ; 0x9c\n", - " 1a0: 239e movs r3, #158 ; 0x9e\n", - " 1a2: 23a0 movs r3, #160 ; 0xa0\n", - " 1a4: 23a2 movs r3, #162 ; 0xa2\n", - " 1a6: 23a4 movs r3, #164 ; 0xa4\n", - " 1a8: 23a6 movs r3, #166 ; 0xa6\n", - " 1aa: 23a8 movs r3, #168 ; 0xa8\n", - " 1ac: 23aa movs r3, #170 ; 0xaa\n", - " 1ae: 23ac movs r3, #172 ; 0xac\n", - " 1b0: 23ae movs r3, #174 ; 0xae\n", - " 1b2: 23b0 movs r3, #176 ; 0xb0\n", - " 1b4: 23b2 movs r3, #178 ; 0xb2\n", - " 1b6: 23b4 movs r3, #180 ; 0xb4\n", - " 1b8: 23b6 movs r3, #182 ; 0xb6\n", - " 1ba: 23b8 movs r3, #184 ; 0xb8\n", - " 1bc: 23ba movs r3, #186 ; 0xba\n", - " 1be: 23bc movs r3, #188 ; 0xbc\n", - " 1c0: 23be movs r3, #190 ; 0xbe\n", - " 1c2: 23c0 movs r3, #192 ; 0xc0\n", - " 1c4: 23c2 movs r3, #194 ; 0xc2\n", - " 1c6: 23c4 movs r3, #196 ; 0xc4\n", - " 1c8: 23c6 movs r3, #198 ; 0xc6\n", - " 1ca: 23c8 movs r3, #200 ; 0xc8\n", - " 1cc: 23ca movs r3, #202 ; 0xca\n", - " 1ce: 23cc movs r3, #204 ; 0xcc\n", - " 1d0: 23ce movs r3, #206 ; 0xce\n", - " 1d2: 23d0 movs r3, #208 ; 0xd0\n", - " 1d4: 23d2 movs r3, #210 ; 0xd2\n", - " 1d6: 23d4 movs r3, #212 ; 0xd4\n", - " 1d8: 23d6 movs r3, #214 ; 0xd6\n", - " 1da: 23d8 movs r3, #216 ; 0xd8\n", - " 1dc: 23da movs r3, #218 ; 0xda\n", - " 1de: 23dc movs r3, #220 ; 0xdc\n", - " 1e0: 23de movs r3, #222 ; 0xde\n", - " 1e2: 23e0 movs r3, #224 ; 0xe0\n", - " 1e4: 23e2 movs r3, #226 ; 0xe2\n", - " 1e6: 23e4 movs r3, #228 ; 0xe4\n", - " 1e8: 23e6 movs r3, #230 ; 0xe6\n", - " 1ea: 23e8 movs r3, #232 ; 0xe8\n", - " 1ec: 23ea movs r3, #234 ; 0xea\n", - " 1ee: 23ec movs r3, #236 ; 0xec\n", - " 1f0: 23ee movs r3, #238 ; 0xee\n", - " 1f2: 23f0 movs r3, #240 ; 0xf0\n", - " 1f4: 23f2 movs r3, #242 ; 0xf2\n", - " 1f6: 23f4 movs r3, #244 ; 0xf4\n", - " 1f8: 23f6 movs r3, #246 ; 0xf6\n", - " 1fa: 23f8 movs r3, #248 ; 0xf8\n", - " 1fc: 23fa movs r3, #250 ; 0xfa\n", - " 1fe: 23fc movs r3, #252 ; 0xfc\n", - " 200: 23fe movs r3, #254 ; 0xfe\n", - " 202: 2300 movs r3, #0\n", - " 204: 2302 movs r3, #2\n", - " 206: 2304 movs r3, #4\n", - " 208: 2306 movs r3, #6\n", - " 20a: 2308 movs r3, #8\n", - " 20c: 230a movs r3, #10\n", - " 20e: 230c movs r3, #12\n", - " 210: 230e movs r3, #14\n", - " 212: 2310 movs r3, #16\n", - " 214: 2312 movs r3, #18\n", - " 216: 2314 movs r3, #20\n", - " 218: 2316 movs r3, #22\n", - " 21a: 2318 movs r3, #24\n", - " 21c: 231a movs r3, #26\n", - " 21e: 231c movs r3, #28\n", - " 220: 231e movs r3, #30\n", - " 222: 2320 movs r3, #32\n", - " 224: 2322 movs r3, #34 ; 0x22\n", - " 226: 2324 movs r3, #36 ; 0x24\n", - " 228: 2326 movs r3, #38 ; 0x26\n", - " 22a: 2328 movs r3, #40 ; 0x28\n", - " 22c: 232a movs r3, #42 ; 0x2a\n", - " 22e: 232c movs r3, #44 ; 0x2c\n", - " 230: 232e movs r3, #46 ; 0x2e\n", - " 232: 2330 movs r3, #48 ; 0x30\n", - " 234: 2332 movs r3, #50 ; 0x32\n", - " 236: 2334 movs r3, #52 ; 0x34\n", - " 238: 2336 movs r3, #54 ; 0x36\n", - " 23a: 2338 movs r3, #56 ; 0x38\n", - " 23c: 233a movs r3, #58 ; 0x3a\n", - " 23e: 233c movs r3, #60 ; 0x3c\n", - " 240: 233e movs r3, #62 ; 0x3e\n", - " 242: 2340 movs r3, #64 ; 0x40\n", - " 244: 2342 movs r3, #66 ; 0x42\n", - " 246: 2344 movs r3, #68 ; 0x44\n", - " 248: 2346 movs r3, #70 ; 0x46\n", - " 24a: 2348 movs r3, #72 ; 0x48\n", - " 24c: 234a movs r3, #74 ; 0x4a\n", - " 24e: 234c movs r3, #76 ; 0x4c\n", - " 250: 234e movs r3, #78 ; 0x4e\n", - " 252: 2350 movs r3, #80 ; 0x50\n", - " 254: 2352 movs r3, #82 ; 0x52\n", - " 256: 2354 movs r3, #84 ; 0x54\n", - " 258: 2356 movs r3, #86 ; 0x56\n", - " 25a: 2358 movs r3, #88 ; 0x58\n", - " 25c: 235a movs r3, #90 ; 0x5a\n", - " 25e: 235c movs r3, #92 ; 0x5c\n", - " 260: 235e movs r3, #94 ; 0x5e\n", - " 262: 2360 movs r3, #96 ; 0x60\n", - " 264: 2362 movs r3, #98 ; 0x62\n", - " 266: 2364 movs r3, #100 ; 0x64\n", - " 268: 2366 movs r3, #102 ; 0x66\n", - " 26a: 2368 movs r3, #104 ; 0x68\n", - " 26c: 236a movs r3, #106 ; 0x6a\n", - " 26e: 236c movs r3, #108 ; 0x6c\n", - " 270: 236e movs r3, #110 ; 0x6e\n", - " 272: 2370 movs r3, #112 ; 0x70\n", - " 274: 2372 movs r3, #114 ; 0x72\n", - " 276: 2374 movs r3, #116 ; 0x74\n", - " 278: 2376 movs r3, #118 ; 0x76\n", - " 27a: 2378 movs r3, #120 ; 0x78\n", - " 27c: 237a movs r3, #122 ; 0x7a\n", - " 27e: 237c movs r3, #124 ; 0x7c\n", - " 280: 237e movs r3, #126 ; 0x7e\n", - " 282: 2380 movs r3, #128 ; 0x80\n", - " 284: 2382 movs r3, #130 ; 0x82\n", - " 286: 2384 movs r3, #132 ; 0x84\n", - " 288: 2386 movs r3, #134 ; 0x86\n", - " 28a: 2388 movs r3, #136 ; 0x88\n", - " 28c: 238a movs r3, #138 ; 0x8a\n", - " 28e: 238c movs r3, #140 ; 0x8c\n", - " 290: 238e movs r3, #142 ; 0x8e\n", - " 292: 2390 movs r3, #144 ; 0x90\n", - " 294: 2392 movs r3, #146 ; 0x92\n", - " 296: 2394 movs r3, #148 ; 0x94\n", - " 298: 2396 movs r3, #150 ; 0x96\n", - " 29a: 2398 movs r3, #152 ; 0x98\n", - " 29c: 239a movs r3, #154 ; 0x9a\n", - " 29e: 239c movs r3, #156 ; 0x9c\n", - " 2a0: 239e movs r3, #158 ; 0x9e\n", - " 2a2: 23a0 movs r3, #160 ; 0xa0\n", - " 2a4: 23a2 movs r3, #162 ; 0xa2\n", - " 2a6: 23a4 movs r3, #164 ; 0xa4\n", - " 2a8: 23a6 movs r3, #166 ; 0xa6\n", - " 2aa: 23a8 movs r3, #168 ; 0xa8\n", - " 2ac: 23aa movs r3, #170 ; 0xaa\n", - " 2ae: 23ac movs r3, #172 ; 0xac\n", - " 2b0: 23ae movs r3, #174 ; 0xae\n", - " 2b2: 23b0 movs r3, #176 ; 0xb0\n", - " 2b4: 23b2 movs r3, #178 ; 0xb2\n", - " 2b6: 23b4 movs r3, #180 ; 0xb4\n", - " 2b8: 23b6 movs r3, #182 ; 0xb6\n", - " 2ba: 23b8 movs r3, #184 ; 0xb8\n", - " 2bc: 23ba movs r3, #186 ; 0xba\n", - " 2be: 23bc movs r3, #188 ; 0xbc\n", - " 2c0: 23be movs r3, #190 ; 0xbe\n", - " 2c2: 23c0 movs r3, #192 ; 0xc0\n", - " 2c4: 23c2 movs r3, #194 ; 0xc2\n", - " 2c6: 23c4 movs r3, #196 ; 0xc4\n", - " 2c8: 23c6 movs r3, #198 ; 0xc6\n", - " 2ca: 23c8 movs r3, #200 ; 0xc8\n", - " 2cc: 23ca movs r3, #202 ; 0xca\n", - " 2ce: 23cc movs r3, #204 ; 0xcc\n", - " 2d0: 23ce movs r3, #206 ; 0xce\n", - " 2d2: 23d0 movs r3, #208 ; 0xd0\n", - " 2d4: 23d2 movs r3, #210 ; 0xd2\n", - " 2d6: 23d4 movs r3, #212 ; 0xd4\n", - " 2d8: 23d6 movs r3, #214 ; 0xd6\n", - " 2da: 23d8 movs r3, #216 ; 0xd8\n", - " 2dc: 23da movs r3, #218 ; 0xda\n", - " 2de: 23dc movs r3, #220 ; 0xdc\n", - " 2e0: 23de movs r3, #222 ; 0xde\n", - " 2e2: 23e0 movs r3, #224 ; 0xe0\n", - " 2e4: 23e2 movs r3, #226 ; 0xe2\n", - " 2e6: 23e4 movs r3, #228 ; 0xe4\n", - " 2e8: 23e6 movs r3, #230 ; 0xe6\n", - " 2ea: 23e8 movs r3, #232 ; 0xe8\n", - " 2ec: 23ea movs r3, #234 ; 0xea\n", - " 2ee: 23ec movs r3, #236 ; 0xec\n", - " 2f0: 23ee movs r3, #238 ; 0xee\n", - " 2f2: 23f0 movs r3, #240 ; 0xf0\n", - " 2f4: 23f2 movs r3, #242 ; 0xf2\n", - " 2f6: 23f4 movs r3, #244 ; 0xf4\n", - " 2f8: 23f6 movs r3, #246 ; 0xf6\n", - " 2fa: 23f8 movs r3, #248 ; 0xf8\n", - " 2fc: 23fa movs r3, #250 ; 0xfa\n", - " 2fe: 23fc movs r3, #252 ; 0xfc\n", - " 300: 23fe movs r3, #254 ; 0xfe\n", - " 302: 2300 movs r3, #0\n", - " 304: 2302 movs r3, #2\n", - " 306: 2304 movs r3, #4\n", - " 308: 2306 movs r3, #6\n", - " 30a: 2308 movs r3, #8\n", - " 30c: 230a movs r3, #10\n", - " 30e: 230c movs r3, #12\n", - " 310: 230e movs r3, #14\n", - " 312: 2310 movs r3, #16\n", - " 314: 2312 movs r3, #18\n", - " 316: 2314 movs r3, #20\n", - " 318: 2316 movs r3, #22\n", - " 31a: 2318 movs r3, #24\n", - " 31c: 231a movs r3, #26\n", - " 31e: 231c movs r3, #28\n", - " 320: 231e movs r3, #30\n", - " 322: 2320 movs r3, #32\n", - " 324: 2322 movs r3, #34 ; 0x22\n", - " 326: 2324 movs r3, #36 ; 0x24\n", - " 328: 2326 movs r3, #38 ; 0x26\n", - " 32a: 2328 movs r3, #40 ; 0x28\n", - " 32c: 232a movs r3, #42 ; 0x2a\n", - " 32e: 232c movs r3, #44 ; 0x2c\n", - " 330: 232e movs r3, #46 ; 0x2e\n", - " 332: 2330 movs r3, #48 ; 0x30\n", - " 334: 2332 movs r3, #50 ; 0x32\n", - " 336: 2334 movs r3, #52 ; 0x34\n", - " 338: 2336 movs r3, #54 ; 0x36\n", - " 33a: 2338 movs r3, #56 ; 0x38\n", - " 33c: 233a movs r3, #58 ; 0x3a\n", - " 33e: 233c movs r3, #60 ; 0x3c\n", - " 340: 233e movs r3, #62 ; 0x3e\n", - " 342: 2340 movs r3, #64 ; 0x40\n", - " 344: 2342 movs r3, #66 ; 0x42\n", - " 346: 2344 movs r3, #68 ; 0x44\n", - " 348: 2346 movs r3, #70 ; 0x46\n", - " 34a: 2348 movs r3, #72 ; 0x48\n", - " 34c: 234a movs r3, #74 ; 0x4a\n", - " 34e: 234c movs r3, #76 ; 0x4c\n", - " 350: 234e movs r3, #78 ; 0x4e\n", - " 352: 2350 movs r3, #80 ; 0x50\n", - " 354: 2352 movs r3, #82 ; 0x52\n", - " 356: 2354 movs r3, #84 ; 0x54\n", - " 358: 2356 movs r3, #86 ; 0x56\n", - " 35a: 2358 movs r3, #88 ; 0x58\n", - " 35c: 235a movs r3, #90 ; 0x5a\n", - " 35e: 235c movs r3, #92 ; 0x5c\n", - " 360: 235e movs r3, #94 ; 0x5e\n", - " 362: 2360 movs r3, #96 ; 0x60\n", - " 364: 2362 movs r3, #98 ; 0x62\n", - " 366: 2364 movs r3, #100 ; 0x64\n", - " 368: 2366 movs r3, #102 ; 0x66\n", - " 36a: 2368 movs r3, #104 ; 0x68\n", - " 36c: 236a movs r3, #106 ; 0x6a\n", - " 36e: 236c movs r3, #108 ; 0x6c\n", - " 370: 236e movs r3, #110 ; 0x6e\n", - " 372: 2370 movs r3, #112 ; 0x70\n", - " 374: 2372 movs r3, #114 ; 0x72\n", - " 376: 2374 movs r3, #116 ; 0x74\n", - " 378: 2376 movs r3, #118 ; 0x76\n", - " 37a: 2378 movs r3, #120 ; 0x78\n", - " 37c: 237a movs r3, #122 ; 0x7a\n", - " 37e: 237c movs r3, #124 ; 0x7c\n", - " 380: 237e movs r3, #126 ; 0x7e\n", - " 382: 2380 movs r3, #128 ; 0x80\n", - " 384: 2382 movs r3, #130 ; 0x82\n", - " 386: 2384 movs r3, #132 ; 0x84\n", - " 388: 2386 movs r3, #134 ; 0x86\n", - " 38a: 2388 movs r3, #136 ; 0x88\n", - " 38c: 238a movs r3, #138 ; 0x8a\n", - " 38e: 238c movs r3, #140 ; 0x8c\n", - " 390: 238e movs r3, #142 ; 0x8e\n", - " 392: 2390 movs r3, #144 ; 0x90\n", - " 394: 2392 movs r3, #146 ; 0x92\n", - " 396: 2394 movs r3, #148 ; 0x94\n", - " 398: 2396 movs r3, #150 ; 0x96\n", - " 39a: 2398 movs r3, #152 ; 0x98\n", - " 39c: 239a movs r3, #154 ; 0x9a\n", - " 39e: 239c movs r3, #156 ; 0x9c\n", - " 3a0: 239e movs r3, #158 ; 0x9e\n", - " 3a2: 23a0 movs r3, #160 ; 0xa0\n", - " 3a4: 23a2 movs r3, #162 ; 0xa2\n", - " 3a6: 23a4 movs r3, #164 ; 0xa4\n", - " 3a8: 23a6 movs r3, #166 ; 0xa6\n", - " 3aa: 23a8 movs r3, #168 ; 0xa8\n", - " 3ac: 23aa movs r3, #170 ; 0xaa\n", - " 3ae: 23ac movs r3, #172 ; 0xac\n", - " 3b0: 23ae movs r3, #174 ; 0xae\n", - " 3b2: 23b0 movs r3, #176 ; 0xb0\n", - " 3b4: 23b2 movs r3, #178 ; 0xb2\n", - " 3b6: 23b4 movs r3, #180 ; 0xb4\n", - " 3b8: 23b6 movs r3, #182 ; 0xb6\n", - " 3ba: 23b8 movs r3, #184 ; 0xb8\n", - " 3bc: 23ba movs r3, #186 ; 0xba\n", - " 3be: 23bc movs r3, #188 ; 0xbc\n", - " 3c0: 23be movs r3, #190 ; 0xbe\n", - " 3c2: 23c0 movs r3, #192 ; 0xc0\n", - " 3c4: 23c2 movs r3, #194 ; 0xc2\n", - " 3c6: 23c4 movs r3, #196 ; 0xc4\n", - " 3c8: 23c6 movs r3, #198 ; 0xc6\n", - " 3ca: 23c8 movs r3, #200 ; 0xc8\n", - " 3cc: 23ca movs r3, #202 ; 0xca\n", - " 3ce: 23cc movs r3, #204 ; 0xcc\n", - " 3d0: 23ce movs r3, #206 ; 0xce\n", - " 3d2: 23d0 movs r3, #208 ; 0xd0\n", - " 3d4: 23d2 movs r3, #210 ; 0xd2\n", - " 3d6: 23d4 movs r3, #212 ; 0xd4\n", - " 3d8: 23d6 movs r3, #214 ; 0xd6\n", - " 3da: 23d8 movs r3, #216 ; 0xd8\n", - " 3dc: 23da movs r3, #218 ; 0xda\n", - " 3de: 23dc movs r3, #220 ; 0xdc\n", - " 3e0: 23de movs r3, #222 ; 0xde\n", - " 3e2: 23e0 movs r3, #224 ; 0xe0\n", - " 3e4: 23e2 movs r3, #226 ; 0xe2\n", - " 3e6: 23e4 movs r3, #228 ; 0xe4\n", - " 3e8: 23e6 movs r3, #230 ; 0xe6\n", - " 3ea: 23e8 movs r3, #232 ; 0xe8\n", - " 3ec: 23ea movs r3, #234 ; 0xea\n", - " 3ee: 23ec movs r3, #236 ; 0xec\n", - " 3f0: 23ee movs r3, #238 ; 0xee\n", - " 3f2: 23f0 movs r3, #240 ; 0xf0\n", - " 3f4: 23f2 movs r3, #242 ; 0xf2\n", - " 3f6: 23f4 movs r3, #244 ; 0xf4\n", - " 3f8: 23f6 movs r3, #246 ; 0xf6\n", - " 3fa: 23f8 movs r3, #248 ; 0xf8\n", - " 3fc: 23fa movs r3, #250 ; 0xfa\n", - " 3fe: 23fc movs r3, #252 ; 0xfc\n", - " 400: 23fe movs r3, #254 ; 0xfe\n", - " 402: 2300 movs r3, #0\n", - " 404: 2302 movs r3, #2\n", - " 406: 2304 movs r3, #4\n", - " 408: 2306 movs r3, #6\n", - " 40a: 2308 movs r3, #8\n", - " 40c: 230a movs r3, #10\n", - " 40e: 230c movs r3, #12\n", - " 410: 230e movs r3, #14\n", - " 412: 2310 movs r3, #16\n", - " 414: 2312 movs r3, #18\n", - " 416: 2314 movs r3, #20\n", - " 418: 2316 movs r3, #22\n", - " 41a: 2318 movs r3, #24\n", - " 41c: 231a movs r3, #26\n", - " 41e: 231c movs r3, #28\n", - " 420: 231e movs r3, #30\n", - " 422: 2320 movs r3, #32\n", - " 424: 2322 movs r3, #34 ; 0x22\n", - " 426: 2324 movs r3, #36 ; 0x24\n", - " 428: 2326 movs r3, #38 ; 0x26\n", - " 42a: 2328 movs r3, #40 ; 0x28\n", - " 42c: 232a movs r3, #42 ; 0x2a\n", - " 42e: 232c movs r3, #44 ; 0x2c\n", - " 430: 232e movs r3, #46 ; 0x2e\n", - " 432: 2330 movs r3, #48 ; 0x30\n", - " 434: 2332 movs r3, #50 ; 0x32\n", - " 436: 2334 movs r3, #52 ; 0x34\n", - " 438: 2336 movs r3, #54 ; 0x36\n", - " 43a: 2338 movs r3, #56 ; 0x38\n", - " 43c: 233a movs r3, #58 ; 0x3a\n", - " 43e: 233c movs r3, #60 ; 0x3c\n", - " 440: 233e movs r3, #62 ; 0x3e\n", - " 442: 2340 movs r3, #64 ; 0x40\n", - " 444: 2342 movs r3, #66 ; 0x42\n", - " 446: 2344 movs r3, #68 ; 0x44\n", - " 448: 2346 movs r3, #70 ; 0x46\n", - " 44a: 2348 movs r3, #72 ; 0x48\n", - " 44c: 234a movs r3, #74 ; 0x4a\n", - " 44e: 234c movs r3, #76 ; 0x4c\n", - " 450: 234e movs r3, #78 ; 0x4e\n", - " 452: 2350 movs r3, #80 ; 0x50\n", - " 454: 2352 movs r3, #82 ; 0x52\n", - " 456: 2354 movs r3, #84 ; 0x54\n", - " 458: 2356 movs r3, #86 ; 0x56\n", - " 45a: 2358 movs r3, #88 ; 0x58\n", - " 45c: 235a movs r3, #90 ; 0x5a\n", - " 45e: 235c movs r3, #92 ; 0x5c\n", - " 460: 235e movs r3, #94 ; 0x5e\n", - " 462: 2360 movs r3, #96 ; 0x60\n", - " 464: 2362 movs r3, #98 ; 0x62\n", - " 466: 2364 movs r3, #100 ; 0x64\n", - " 468: 2366 movs r3, #102 ; 0x66\n", - " 46a: 2368 movs r3, #104 ; 0x68\n", - " 46c: 236a movs r3, #106 ; 0x6a\n", - " 46e: 236c movs r3, #108 ; 0x6c\n", - " 470: 236e movs r3, #110 ; 0x6e\n", - " 472: 2370 movs r3, #112 ; 0x70\n", - " 474: 2372 movs r3, #114 ; 0x72\n", - " 476: 2374 movs r3, #116 ; 0x74\n", - " 478: 2376 movs r3, #118 ; 0x76\n", - " 47a: 2378 movs r3, #120 ; 0x78\n", - " 47c: 237a movs r3, #122 ; 0x7a\n", - " 47e: 237c movs r3, #124 ; 0x7c\n", - " 480: 237e movs r3, #126 ; 0x7e\n", - " 482: 2380 movs r3, #128 ; 0x80\n", - " 484: 2382 movs r3, #130 ; 0x82\n", - " 486: 2384 movs r3, #132 ; 0x84\n", - " 488: 2386 movs r3, #134 ; 0x86\n", - " 48a: 2388 movs r3, #136 ; 0x88\n", - " 48c: 238a movs r3, #138 ; 0x8a\n", - " 48e: 238c movs r3, #140 ; 0x8c\n", - " 490: 238e movs r3, #142 ; 0x8e\n", - " 492: 2390 movs r3, #144 ; 0x90\n", - " 494: 2392 movs r3, #146 ; 0x92\n", - " 496: 2394 movs r3, #148 ; 0x94\n", - " 498: 2396 movs r3, #150 ; 0x96\n", - " 49a: 2398 movs r3, #152 ; 0x98\n", - " 49c: 239a movs r3, #154 ; 0x9a\n", - " 49e: 239c movs r3, #156 ; 0x9c\n", - " 4a0: 239e movs r3, #158 ; 0x9e\n", - " 4a2: 23a0 movs r3, #160 ; 0xa0\n", - " 4a4: 23a2 movs r3, #162 ; 0xa2\n", - " 4a6: 23a4 movs r3, #164 ; 0xa4\n", - " 4a8: 23a6 movs r3, #166 ; 0xa6\n", - " 4aa: 23a8 movs r3, #168 ; 0xa8\n", - " 4ac: 23aa movs r3, #170 ; 0xaa\n", - " 4ae: 23ac movs r3, #172 ; 0xac\n", - " 4b0: 23ae movs r3, #174 ; 0xae\n", - " 4b2: 23b0 movs r3, #176 ; 0xb0\n", - " 4b4: 23b2 movs r3, #178 ; 0xb2\n", - " 4b6: 23b4 movs r3, #180 ; 0xb4\n", - " 4b8: 23b6 movs r3, #182 ; 0xb6\n", - " 4ba: 23b8 movs r3, #184 ; 0xb8\n", - " 4bc: 23ba movs r3, #186 ; 0xba\n", - " 4be: 23bc movs r3, #188 ; 0xbc\n", - " 4c0: 23be movs r3, #190 ; 0xbe\n", - " 4c2: 23c0 movs r3, #192 ; 0xc0\n", - " 4c4: 23c2 movs r3, #194 ; 0xc2\n", - " 4c6: 23c4 movs r3, #196 ; 0xc4\n", - " 4c8: 23c6 movs r3, #198 ; 0xc6\n", - " 4ca: 23c8 movs r3, #200 ; 0xc8\n", - " 4cc: 23ca movs r3, #202 ; 0xca\n", - " 4ce: 23cc movs r3, #204 ; 0xcc\n", - " 4d0: 23ce movs r3, #206 ; 0xce\n", - " 4d2: 23d0 movs r3, #208 ; 0xd0\n", - " 4d4: 23d2 movs r3, #210 ; 0xd2\n", - " 4d6: 23d4 movs r3, #212 ; 0xd4\n", - " 4d8: 23d6 movs r3, #214 ; 0xd6\n", - " 4da: 23d8 movs r3, #216 ; 0xd8\n", - " 4dc: 23da movs r3, #218 ; 0xda\n", - " 4de: 23dc movs r3, #220 ; 0xdc\n", - " 4e0: 23de movs r3, #222 ; 0xde\n", - " 4e2: 23e0 movs r3, #224 ; 0xe0\n", - " 4e4: 23e2 movs r3, #226 ; 0xe2\n", - " 4e6: 23e4 movs r3, #228 ; 0xe4\n", - " 4e8: 23e6 movs r3, #230 ; 0xe6\n", - " 4ea: 23e8 movs r3, #232 ; 0xe8\n", - " 4ec: 23ea movs r3, #234 ; 0xea\n", - " 4ee: 23ec movs r3, #236 ; 0xec\n", - " 4f0: 23ee movs r3, #238 ; 0xee\n", - " 4f2: 23f0 movs r3, #240 ; 0xf0\n", - " 4f4: 23f2 movs r3, #242 ; 0xf2\n", - " 4f6: 23f4 movs r3, #244 ; 0xf4\n", - " 4f8: 23f6 movs r3, #246 ; 0xf6\n", - " 4fa: 23f8 movs r3, #248 ; 0xf8\n", - " 4fc: 23fa movs r3, #250 ; 0xfa\n", - " 4fe: 23fc movs r3, #252 ; 0xfc\n", - " 500: 23fe movs r3, #254 ; 0xfe\n", - " 502: 2300 movs r3, #0\n", - " 504: 2302 movs r3, #2\n", - " 506: 2304 movs r3, #4\n", - " 508: 2306 movs r3, #6\n", - " 50a: 2308 movs r3, #8\n", - " 50c: 230a movs r3, #10\n", - " 50e: 230c movs r3, #12\n", - " 510: 230e movs r3, #14\n", - " 512: 2310 movs r3, #16\n", - " 514: 2312 movs r3, #18\n", - " 516: 2314 movs r3, #20\n", - " 518: 2316 movs r3, #22\n", - " 51a: 2318 movs r3, #24\n", - " 51c: 231a movs r3, #26\n", - " 51e: 231c movs r3, #28\n", - " 520: 231e movs r3, #30\n", - " 522: 2320 movs r3, #32\n", - " 524: 2322 movs r3, #34 ; 0x22\n", - " 526: 2324 movs r3, #36 ; 0x24\n", - " 528: 2326 movs r3, #38 ; 0x26\n", - " 52a: 2328 movs r3, #40 ; 0x28\n", - " 52c: 232a movs r3, #42 ; 0x2a\n", - " 52e: 232c movs r3, #44 ; 0x2c\n", - " 530: 232e movs r3, #46 ; 0x2e\n", - " 532: 2330 movs r3, #48 ; 0x30\n", - " 534: 2332 movs r3, #50 ; 0x32\n", - " 536: 2334 movs r3, #52 ; 0x34\n", - " 538: 2336 movs r3, #54 ; 0x36\n", - " 53a: 2338 movs r3, #56 ; 0x38\n", - " 53c: 233a movs r3, #58 ; 0x3a\n", - " 53e: 233c movs r3, #60 ; 0x3c\n", - " 540: 233e movs r3, #62 ; 0x3e\n", - " 542: 2340 movs r3, #64 ; 0x40\n", - " 544: 2342 movs r3, #66 ; 0x42\n", - " 546: 2344 movs r3, #68 ; 0x44\n", - " 548: 2346 movs r3, #70 ; 0x46\n", - " 54a: 2348 movs r3, #72 ; 0x48\n", - " 54c: 234a movs r3, #74 ; 0x4a\n", - " 54e: 234c movs r3, #76 ; 0x4c\n", - " 550: 234e movs r3, #78 ; 0x4e\n", - " 552: 2350 movs r3, #80 ; 0x50\n", - " 554: 2352 movs r3, #82 ; 0x52\n", - " 556: 2354 movs r3, #84 ; 0x54\n", - " 558: 2356 movs r3, #86 ; 0x56\n", - " 55a: 2358 movs r3, #88 ; 0x58\n", - " 55c: 235a movs r3, #90 ; 0x5a\n", - " 55e: 235c movs r3, #92 ; 0x5c\n", - " 560: 235e movs r3, #94 ; 0x5e\n", - " 562: 2360 movs r3, #96 ; 0x60\n", - " 564: 2362 movs r3, #98 ; 0x62\n", - " 566: 2364 movs r3, #100 ; 0x64\n", - " 568: 2366 movs r3, #102 ; 0x66\n", - " 56a: 2368 movs r3, #104 ; 0x68\n", - " 56c: 236a movs r3, #106 ; 0x6a\n", - " 56e: 236c movs r3, #108 ; 0x6c\n", - " 570: 236e movs r3, #110 ; 0x6e\n", - " 572: 2370 movs r3, #112 ; 0x70\n", - " 574: 2372 movs r3, #114 ; 0x72\n", - " 576: 2374 movs r3, #116 ; 0x74\n", - " 578: 2376 movs r3, #118 ; 0x76\n", - " 57a: 2378 movs r3, #120 ; 0x78\n", - " 57c: 237a movs r3, #122 ; 0x7a\n", - " 57e: 237c movs r3, #124 ; 0x7c\n", - " 580: 237e movs r3, #126 ; 0x7e\n", - " 582: 2380 movs r3, #128 ; 0x80\n", - " 584: 2382 movs r3, #130 ; 0x82\n", - " 586: 2384 movs r3, #132 ; 0x84\n", - " 588: 2386 movs r3, #134 ; 0x86\n", - " 58a: 2388 movs r3, #136 ; 0x88\n", - " 58c: 238a movs r3, #138 ; 0x8a\n", - " 58e: 238c movs r3, #140 ; 0x8c\n", - " 590: 238e movs r3, #142 ; 0x8e\n", - " 592: 2390 movs r3, #144 ; 0x90\n", - " 594: 2392 movs r3, #146 ; 0x92\n", - " 596: 2394 movs r3, #148 ; 0x94\n", - " 598: 2396 movs r3, #150 ; 0x96\n", - " 59a: 2398 movs r3, #152 ; 0x98\n", - " 59c: 239a movs r3, #154 ; 0x9a\n", - " 59e: 239c movs r3, #156 ; 0x9c\n", - " 5a0: 239e movs r3, #158 ; 0x9e\n", - " 5a2: 23a0 movs r3, #160 ; 0xa0\n", - " 5a4: 23a2 movs r3, #162 ; 0xa2\n", - " 5a6: 23a4 movs r3, #164 ; 0xa4\n", - " 5a8: 23a6 movs r3, #166 ; 0xa6\n", - " 5aa: 23a8 movs r3, #168 ; 0xa8\n", - " 5ac: 23aa movs r3, #170 ; 0xaa\n", - " 5ae: 23ac movs r3, #172 ; 0xac\n", - " 5b0: 23ae movs r3, #174 ; 0xae\n", - " 5b2: 23b0 movs r3, #176 ; 0xb0\n", - " 5b4: 23b2 movs r3, #178 ; 0xb2\n", - " 5b6: 23b4 movs r3, #180 ; 0xb4\n", - " 5b8: 23b6 movs r3, #182 ; 0xb6\n", - " 5ba: 23b8 movs r3, #184 ; 0xb8\n", - " 5bc: 23ba movs r3, #186 ; 0xba\n", - " 5be: 23bc movs r3, #188 ; 0xbc\n", - " 5c0: 23be movs r3, #190 ; 0xbe\n", - " 5c2: 23c0 movs r3, #192 ; 0xc0\n", - " 5c4: 23c2 movs r3, #194 ; 0xc2\n", - " 5c6: 23c4 movs r3, #196 ; 0xc4\n", - " 5c8: 23c6 movs r3, #198 ; 0xc6\n", - " 5ca: 23c8 movs r3, #200 ; 0xc8\n", - " 5cc: 23ca movs r3, #202 ; 0xca\n", - " 5ce: 23cc movs r3, #204 ; 0xcc\n", - " 5d0: 23ce movs r3, #206 ; 0xce\n", - " 5d2: 23d0 movs r3, #208 ; 0xd0\n", - " 5d4: 23d2 movs r3, #210 ; 0xd2\n", - " 5d6: 23d4 movs r3, #212 ; 0xd4\n", - " 5d8: 23d6 movs r3, #214 ; 0xd6\n", - " 5da: 23d8 movs r3, #216 ; 0xd8\n", - " 5dc: 23da movs r3, #218 ; 0xda\n", - " 5de: 23dc movs r3, #220 ; 0xdc\n", - " 5e0: 23de movs r3, #222 ; 0xde\n", - " 5e2: 23e0 movs r3, #224 ; 0xe0\n", - " 5e4: 23e2 movs r3, #226 ; 0xe2\n", - " 5e6: 23e4 movs r3, #228 ; 0xe4\n", - " 5e8: 23e6 movs r3, #230 ; 0xe6\n", - " 5ea: 23e8 movs r3, #232 ; 0xe8\n", - " 5ec: 23ea movs r3, #234 ; 0xea\n", - " 5ee: 23ec movs r3, #236 ; 0xec\n", - " 5f0: 23ee movs r3, #238 ; 0xee\n", - " 5f2: 23f0 movs r3, #240 ; 0xf0\n", - " 5f4: 23f2 movs r3, #242 ; 0xf2\n", - " 5f6: 23f4 movs r3, #244 ; 0xf4\n", - " 5f8: 23f6 movs r3, #246 ; 0xf6\n", - " 5fa: 23f8 movs r3, #248 ; 0xf8\n", - " 5fc: 23fa movs r3, #250 ; 0xfa\n", - " 5fe: 23fc movs r3, #252 ; 0xfc\n", - " 600: 23fe movs r3, #254 ; 0xfe\n", - " 602: 2300 movs r3, #0\n", - " 604: 2302 movs r3, #2\n", - " 606: 2304 movs r3, #4\n", - " 608: 2306 movs r3, #6\n", - " 60a: 2308 movs r3, #8\n", - " 60c: 230a movs r3, #10\n", - " 60e: 230c movs r3, #12\n", - " 610: 230e movs r3, #14\n", - " 612: 2310 movs r3, #16\n", - " 614: 2312 movs r3, #18\n", - " 616: 2314 movs r3, #20\n", - " 618: 2316 movs r3, #22\n", - " 61a: 2318 movs r3, #24\n", - " 61c: 231a movs r3, #26\n", - " 61e: 231c movs r3, #28\n", - " 620: 231e movs r3, #30\n", - " 622: 2320 movs r3, #32\n", - " 624: 2322 movs r3, #34 ; 0x22\n", - " 626: 2324 movs r3, #36 ; 0x24\n", - " 628: 2326 movs r3, #38 ; 0x26\n", - " 62a: 2328 movs r3, #40 ; 0x28\n", - " 62c: 232a movs r3, #42 ; 0x2a\n", - " 62e: 232c movs r3, #44 ; 0x2c\n", - " 630: 232e movs r3, #46 ; 0x2e\n", - " 632: 2330 movs r3, #48 ; 0x30\n", - " 634: 2332 movs r3, #50 ; 0x32\n", - " 636: 2334 movs r3, #52 ; 0x34\n", - " 638: 2336 movs r3, #54 ; 0x36\n", - " 63a: 2338 movs r3, #56 ; 0x38\n", - " 63c: 233a movs r3, #58 ; 0x3a\n", - " 63e: 233c movs r3, #60 ; 0x3c\n", - " 640: 233e movs r3, #62 ; 0x3e\n", - " 642: 2340 movs r3, #64 ; 0x40\n", - " 644: 2342 movs r3, #66 ; 0x42\n", - " 646: 2344 movs r3, #68 ; 0x44\n", - " 648: 2346 movs r3, #70 ; 0x46\n", - " 64a: 2348 movs r3, #72 ; 0x48\n", - " 64c: 234a movs r3, #74 ; 0x4a\n", - " 64e: 234c movs r3, #76 ; 0x4c\n", - " 650: 234e movs r3, #78 ; 0x4e\n", - " 652: 2350 movs r3, #80 ; 0x50\n", - " 654: 2352 movs r3, #82 ; 0x52\n", - " 656: 2354 movs r3, #84 ; 0x54\n", - " 658: 2356 movs r3, #86 ; 0x56\n", - " 65a: 2358 movs r3, #88 ; 0x58\n", - " 65c: 235a movs r3, #90 ; 0x5a\n", - " 65e: 235c movs r3, #92 ; 0x5c\n", - " 660: 235e movs r3, #94 ; 0x5e\n", - " 662: 2360 movs r3, #96 ; 0x60\n", - " 664: 2362 movs r3, #98 ; 0x62\n", - " 666: 2364 movs r3, #100 ; 0x64\n", - " 668: 2366 movs r3, #102 ; 0x66\n", - " 66a: 2368 movs r3, #104 ; 0x68\n", - " 66c: 236a movs r3, #106 ; 0x6a\n", - " 66e: 236c movs r3, #108 ; 0x6c\n", - " 670: 236e movs r3, #110 ; 0x6e\n", - " 672: 2370 movs r3, #112 ; 0x70\n", - " 674: 2372 movs r3, #114 ; 0x72\n", - " 676: 2374 movs r3, #116 ; 0x74\n", - " 678: 2376 movs r3, #118 ; 0x76\n", - " 67a: 2378 movs r3, #120 ; 0x78\n", - " 67c: 237a movs r3, #122 ; 0x7a\n", - " 67e: 237c movs r3, #124 ; 0x7c\n", - " 680: 237e movs r3, #126 ; 0x7e\n", - " 682: 2380 movs r3, #128 ; 0x80\n", - " 684: 2382 movs r3, #130 ; 0x82\n", - " 686: 2384 movs r3, #132 ; 0x84\n", - " 688: 2386 movs r3, #134 ; 0x86\n", - " 68a: 2388 movs r3, #136 ; 0x88\n", - " 68c: 238a movs r3, #138 ; 0x8a\n", - " 68e: 238c movs r3, #140 ; 0x8c\n", - " 690: 238e movs r3, #142 ; 0x8e\n", - " 692: 2390 movs r3, #144 ; 0x90\n", - " 694: 2392 movs r3, #146 ; 0x92\n", - " 696: 2394 movs r3, #148 ; 0x94\n", - " 698: 2396 movs r3, #150 ; 0x96\n", - " 69a: 2398 movs r3, #152 ; 0x98\n", - " 69c: 239a movs r3, #154 ; 0x9a\n", - " 69e: 239c movs r3, #156 ; 0x9c\n", - " 6a0: 239e movs r3, #158 ; 0x9e\n", - " 6a2: 23a0 movs r3, #160 ; 0xa0\n", - " 6a4: 23a2 movs r3, #162 ; 0xa2\n", - " 6a6: 23a4 movs r3, #164 ; 0xa4\n", - " 6a8: 23a6 movs r3, #166 ; 0xa6\n", - " 6aa: 23a8 movs r3, #168 ; 0xa8\n", - " 6ac: 23aa movs r3, #170 ; 0xaa\n", - " 6ae: 23ac movs r3, #172 ; 0xac\n", - " 6b0: 23ae movs r3, #174 ; 0xae\n", - " 6b2: 23b0 movs r3, #176 ; 0xb0\n", - " 6b4: 23b2 movs r3, #178 ; 0xb2\n", - " 6b6: 23b4 movs r3, #180 ; 0xb4\n", - " 6b8: 23b6 movs r3, #182 ; 0xb6\n", - " 6ba: 23b8 movs r3, #184 ; 0xb8\n", - " 6bc: 23ba movs r3, #186 ; 0xba\n", - " 6be: 23bc movs r3, #188 ; 0xbc\n", - " 6c0: 23be movs r3, #190 ; 0xbe\n", - " 6c2: 23c0 movs r3, #192 ; 0xc0\n", - " 6c4: 23c2 movs r3, #194 ; 0xc2\n", - " 6c6: 23c4 movs r3, #196 ; 0xc4\n", - " 6c8: 23c6 movs r3, #198 ; 0xc6\n", - " 6ca: 23c8 movs r3, #200 ; 0xc8\n", - " 6cc: 23ca movs r3, #202 ; 0xca\n", - " 6ce: 23cc movs r3, #204 ; 0xcc\n", - " 6d0: 23ce movs r3, #206 ; 0xce\n", - " 6d2: 23d0 movs r3, #208 ; 0xd0\n", - " 6d4: 23d2 movs r3, #210 ; 0xd2\n", - " 6d6: 23d4 movs r3, #212 ; 0xd4\n", - " 6d8: 23d6 movs r3, #214 ; 0xd6\n", - " 6da: 23d8 movs r3, #216 ; 0xd8\n", - " 6dc: 23da movs r3, #218 ; 0xda\n", - " 6de: 23dc movs r3, #220 ; 0xdc\n", - " 6e0: 23de movs r3, #222 ; 0xde\n", - " 6e2: 23e0 movs r3, #224 ; 0xe0\n", - " 6e4: 23e2 movs r3, #226 ; 0xe2\n", - " 6e6: 23e4 movs r3, #228 ; 0xe4\n", - " 6e8: 23e6 movs r3, #230 ; 0xe6\n", - " 6ea: 23e8 movs r3, #232 ; 0xe8\n", - " 6ec: 23ea movs r3, #234 ; 0xea\n", - " 6ee: 23ec movs r3, #236 ; 0xec\n", - " 6f0: 23ee movs r3, #238 ; 0xee\n", - " 6f2: 23f0 movs r3, #240 ; 0xf0\n", - " 6f4: 23f2 movs r3, #242 ; 0xf2\n", - " 6f6: 23f4 movs r3, #244 ; 0xf4\n", - " 6f8: 23f6 movs r3, #246 ; 0xf6\n", - " 6fa: 23f8 movs r3, #248 ; 0xf8\n", - " 6fc: 23fa movs r3, #250 ; 0xfa\n", - " 6fe: 23fc movs r3, #252 ; 0xfc\n", - " 700: 23fe movs r3, #254 ; 0xfe\n", - " 702: 2300 movs r3, #0\n", - " 704: 2302 movs r3, #2\n", - " 706: 2304 movs r3, #4\n", - " 708: 2306 movs r3, #6\n", - " 70a: 2308 movs r3, #8\n", - " 70c: 230a movs r3, #10\n", - " 70e: 230c movs r3, #12\n", - " 710: 230e movs r3, #14\n", - " 712: 2310 movs r3, #16\n", - " 714: 2312 movs r3, #18\n", - " 716: 2314 movs r3, #20\n", - " 718: 2316 movs r3, #22\n", - " 71a: 2318 movs r3, #24\n", - " 71c: 231a movs r3, #26\n", - " 71e: 231c movs r3, #28\n", - " 720: 231e movs r3, #30\n", - " 722: 2320 movs r3, #32\n", - " 724: 2322 movs r3, #34 ; 0x22\n", - " 726: 2324 movs r3, #36 ; 0x24\n", - " 728: 2326 movs r3, #38 ; 0x26\n", - " 72a: 2328 movs r3, #40 ; 0x28\n", - " 72c: 232a movs r3, #42 ; 0x2a\n", - " 72e: 232c movs r3, #44 ; 0x2c\n", - " 730: 232e movs r3, #46 ; 0x2e\n", - " 732: 2330 movs r3, #48 ; 0x30\n", - " 734: 2332 movs r3, #50 ; 0x32\n", - " 736: 2334 movs r3, #52 ; 0x34\n", - " 738: 2336 movs r3, #54 ; 0x36\n", - " 73a: 2338 movs r3, #56 ; 0x38\n", - " 73c: 233a movs r3, #58 ; 0x3a\n", - " 73e: 233c movs r3, #60 ; 0x3c\n", - " 740: 233e movs r3, #62 ; 0x3e\n", - " 742: 2340 movs r3, #64 ; 0x40\n", - " 744: 2342 movs r3, #66 ; 0x42\n", - " 746: 2344 movs r3, #68 ; 0x44\n", - " 748: 2346 movs r3, #70 ; 0x46\n", - " 74a: 2348 movs r3, #72 ; 0x48\n", - " 74c: 234a movs r3, #74 ; 0x4a\n", - " 74e: 234c movs r3, #76 ; 0x4c\n", - " 750: 234e movs r3, #78 ; 0x4e\n", - " 752: 2350 movs r3, #80 ; 0x50\n", - " 754: 2352 movs r3, #82 ; 0x52\n", - " 756: 2354 movs r3, #84 ; 0x54\n", - " 758: 2356 movs r3, #86 ; 0x56\n", - " 75a: 2358 movs r3, #88 ; 0x58\n", - " 75c: 235a movs r3, #90 ; 0x5a\n", - " 75e: 235c movs r3, #92 ; 0x5c\n", - " 760: 235e movs r3, #94 ; 0x5e\n", - " 762: 2360 movs r3, #96 ; 0x60\n", - " 764: 2362 movs r3, #98 ; 0x62\n", - " 766: 2364 movs r3, #100 ; 0x64\n", - " 768: 2366 movs r3, #102 ; 0x66\n", - " 76a: 2368 movs r3, #104 ; 0x68\n", - " 76c: 236a movs r3, #106 ; 0x6a\n", - " 76e: 236c movs r3, #108 ; 0x6c\n", - " 770: 236e movs r3, #110 ; 0x6e\n", - " 772: 2370 movs r3, #112 ; 0x70\n", - " 774: 2372 movs r3, #114 ; 0x72\n", - " 776: 2374 movs r3, #116 ; 0x74\n", - " 778: 2376 movs r3, #118 ; 0x76\n", - " 77a: 2378 movs r3, #120 ; 0x78\n", - " 77c: 237a movs r3, #122 ; 0x7a\n", - " 77e: 237c movs r3, #124 ; 0x7c\n", - " 780: 237e movs r3, #126 ; 0x7e\n", - " 782: 2380 movs r3, #128 ; 0x80\n", - " 784: 2382 movs r3, #130 ; 0x82\n", - " 786: 2384 movs r3, #132 ; 0x84\n", - " 788: 2386 movs r3, #134 ; 0x86\n", - " 78a: 2388 movs r3, #136 ; 0x88\n", - " 78c: 238a movs r3, #138 ; 0x8a\n", - " 78e: 238c movs r3, #140 ; 0x8c\n", - " 790: 238e movs r3, #142 ; 0x8e\n", - " 792: 2390 movs r3, #144 ; 0x90\n", - " 794: 2392 movs r3, #146 ; 0x92\n", - " 796: 2394 movs r3, #148 ; 0x94\n", - " 798: 2396 movs r3, #150 ; 0x96\n", - " 79a: 2398 movs r3, #152 ; 0x98\n", - " 79c: 239a movs r3, #154 ; 0x9a\n", - " 79e: 239c movs r3, #156 ; 0x9c\n", - " 7a0: 239e movs r3, #158 ; 0x9e\n", - " 7a2: 23a0 movs r3, #160 ; 0xa0\n", - " 7a4: 23a2 movs r3, #162 ; 0xa2\n", - " 7a6: 23a4 movs r3, #164 ; 0xa4\n", - " 7a8: 23a6 movs r3, #166 ; 0xa6\n", - " 7aa: 23a8 movs r3, #168 ; 0xa8\n", - " 7ac: 23aa movs r3, #170 ; 0xaa\n", - " 7ae: 23ac movs r3, #172 ; 0xac\n", - " 7b0: 23ae movs r3, #174 ; 0xae\n", - " 7b2: 23b0 movs r3, #176 ; 0xb0\n", - " 7b4: 23b2 movs r3, #178 ; 0xb2\n", - " 7b6: 23b4 movs r3, #180 ; 0xb4\n", - " 7b8: 23b6 movs r3, #182 ; 0xb6\n", - " 7ba: 23b8 movs r3, #184 ; 0xb8\n", - " 7bc: 23ba movs r3, #186 ; 0xba\n", - " 7be: 23bc movs r3, #188 ; 0xbc\n", - " 7c0: 23be movs r3, #190 ; 0xbe\n", - " 7c2: 23c0 movs r3, #192 ; 0xc0\n", - " 7c4: 23c2 movs r3, #194 ; 0xc2\n", - " 7c6: 23c4 movs r3, #196 ; 0xc4\n", - " 7c8: 23c6 movs r3, #198 ; 0xc6\n", - " 7ca: 23c8 movs r3, #200 ; 0xc8\n", - " 7cc: 23ca movs r3, #202 ; 0xca\n", - " 7ce: 23cc movs r3, #204 ; 0xcc\n", - " 7d0: 23ce movs r3, #206 ; 0xce\n", - " 7d2: 23d0 movs r3, #208 ; 0xd0\n", - " 7d4: 23d2 movs r3, #210 ; 0xd2\n", - " 7d6: 23d4 movs r3, #212 ; 0xd4\n", - " 7d8: 23d6 movs r3, #214 ; 0xd6\n", - " 7da: 23d8 movs r3, #216 ; 0xd8\n", - " 7dc: 23da movs r3, #218 ; 0xda\n", - " 7de: 23dc movs r3, #220 ; 0xdc\n", - " 7e0: 23de movs r3, #222 ; 0xde\n", - " 7e2: 23e0 movs r3, #224 ; 0xe0\n", - " 7e4: 23e2 movs r3, #226 ; 0xe2\n", - " 7e6: 23e4 movs r3, #228 ; 0xe4\n", - " 7e8: 23e6 movs r3, #230 ; 0xe6\n", - " 7ea: 23e8 movs r3, #232 ; 0xe8\n", - " 7ec: 23ea movs r3, #234 ; 0xea\n", - " 7ee: 23ec movs r3, #236 ; 0xec\n", - " 7f0: 23ee movs r3, #238 ; 0xee\n", - " 7f2: 23f0 movs r3, #240 ; 0xf0\n", - " 7f4: 23f2 movs r3, #242 ; 0xf2\n", - " 7f6: 23f4 movs r3, #244 ; 0xf4\n", - " 7f8: 23f6 movs r3, #246 ; 0xf6\n", - " 7fa: 23f8 movs r3, #248 ; 0xf8\n", - " 7fc: 23fa movs r3, #250 ; 0xfa\n", - " 7fe: 23fc movs r3, #252 ; 0xfc\n", - " 800: 23fe movs r3, #254 ; 0xfe\n", - " 802: 4611 mov r1, r2\n", - nullptr -}; -const char* const Branch32Results[] = { - " 0: f000 bc01 b.w 806 <Branch32+0x806>\n", - " 4: 2300 movs r3, #0\n", - " 6: 2302 movs r3, #2\n", - " 8: 2304 movs r3, #4\n", - " a: 2306 movs r3, #6\n", - " c: 2308 movs r3, #8\n", - " e: 230a movs r3, #10\n", - " 10: 230c movs r3, #12\n", - " 12: 230e movs r3, #14\n", - " 14: 2310 movs r3, #16\n", - " 16: 2312 movs r3, #18\n", - " 18: 2314 movs r3, #20\n", - " 1a: 2316 movs r3, #22\n", - " 1c: 2318 movs r3, #24\n", - " 1e: 231a movs r3, #26\n", - " 20: 231c movs r3, #28\n", - " 22: 231e movs r3, #30\n", - " 24: 2320 movs r3, #32\n", - " 26: 2322 movs r3, #34 ; 0x22\n", - " 28: 2324 movs r3, #36 ; 0x24\n", - " 2a: 2326 movs r3, #38 ; 0x26\n", - " 2c: 2328 movs r3, #40 ; 0x28\n", - " 2e: 232a movs r3, #42 ; 0x2a\n", - " 30: 232c movs r3, #44 ; 0x2c\n", - " 32: 232e movs r3, #46 ; 0x2e\n", - " 34: 2330 movs r3, #48 ; 0x30\n", - " 36: 2332 movs r3, #50 ; 0x32\n", - " 38: 2334 movs r3, #52 ; 0x34\n", - " 3a: 2336 movs r3, #54 ; 0x36\n", - " 3c: 2338 movs r3, #56 ; 0x38\n", - " 3e: 233a movs r3, #58 ; 0x3a\n", - " 40: 233c movs r3, #60 ; 0x3c\n", - " 42: 233e movs r3, #62 ; 0x3e\n", - " 44: 2340 movs r3, #64 ; 0x40\n", - " 46: 2342 movs r3, #66 ; 0x42\n", - " 48: 2344 movs r3, #68 ; 0x44\n", - " 4a: 2346 movs r3, #70 ; 0x46\n", - " 4c: 2348 movs r3, #72 ; 0x48\n", - " 4e: 234a movs r3, #74 ; 0x4a\n", - " 50: 234c movs r3, #76 ; 0x4c\n", - " 52: 234e movs r3, #78 ; 0x4e\n", - " 54: 2350 movs r3, #80 ; 0x50\n", - " 56: 2352 movs r3, #82 ; 0x52\n", - " 58: 2354 movs r3, #84 ; 0x54\n", - " 5a: 2356 movs r3, #86 ; 0x56\n", - " 5c: 2358 movs r3, #88 ; 0x58\n", - " 5e: 235a movs r3, #90 ; 0x5a\n", - " 60: 235c movs r3, #92 ; 0x5c\n", - " 62: 235e movs r3, #94 ; 0x5e\n", - " 64: 2360 movs r3, #96 ; 0x60\n", - " 66: 2362 movs r3, #98 ; 0x62\n", - " 68: 2364 movs r3, #100 ; 0x64\n", - " 6a: 2366 movs r3, #102 ; 0x66\n", - " 6c: 2368 movs r3, #104 ; 0x68\n", - " 6e: 236a movs r3, #106 ; 0x6a\n", - " 70: 236c movs r3, #108 ; 0x6c\n", - " 72: 236e movs r3, #110 ; 0x6e\n", - " 74: 2370 movs r3, #112 ; 0x70\n", - " 76: 2372 movs r3, #114 ; 0x72\n", - " 78: 2374 movs r3, #116 ; 0x74\n", - " 7a: 2376 movs r3, #118 ; 0x76\n", - " 7c: 2378 movs r3, #120 ; 0x78\n", - " 7e: 237a movs r3, #122 ; 0x7a\n", - " 80: 237c movs r3, #124 ; 0x7c\n", - " 82: 237e movs r3, #126 ; 0x7e\n", - " 84: 2380 movs r3, #128 ; 0x80\n", - " 86: 2382 movs r3, #130 ; 0x82\n", - " 88: 2384 movs r3, #132 ; 0x84\n", - " 8a: 2386 movs r3, #134 ; 0x86\n", - " 8c: 2388 movs r3, #136 ; 0x88\n", - " 8e: 238a movs r3, #138 ; 0x8a\n", - " 90: 238c movs r3, #140 ; 0x8c\n", - " 92: 238e movs r3, #142 ; 0x8e\n", - " 94: 2390 movs r3, #144 ; 0x90\n", - " 96: 2392 movs r3, #146 ; 0x92\n", - " 98: 2394 movs r3, #148 ; 0x94\n", - " 9a: 2396 movs r3, #150 ; 0x96\n", - " 9c: 2398 movs r3, #152 ; 0x98\n", - " 9e: 239a movs r3, #154 ; 0x9a\n", - " a0: 239c movs r3, #156 ; 0x9c\n", - " a2: 239e movs r3, #158 ; 0x9e\n", - " a4: 23a0 movs r3, #160 ; 0xa0\n", - " a6: 23a2 movs r3, #162 ; 0xa2\n", - " a8: 23a4 movs r3, #164 ; 0xa4\n", - " aa: 23a6 movs r3, #166 ; 0xa6\n", - " ac: 23a8 movs r3, #168 ; 0xa8\n", - " ae: 23aa movs r3, #170 ; 0xaa\n", - " b0: 23ac movs r3, #172 ; 0xac\n", - " b2: 23ae movs r3, #174 ; 0xae\n", - " b4: 23b0 movs r3, #176 ; 0xb0\n", - " b6: 23b2 movs r3, #178 ; 0xb2\n", - " b8: 23b4 movs r3, #180 ; 0xb4\n", - " ba: 23b6 movs r3, #182 ; 0xb6\n", - " bc: 23b8 movs r3, #184 ; 0xb8\n", - " be: 23ba movs r3, #186 ; 0xba\n", - " c0: 23bc movs r3, #188 ; 0xbc\n", - " c2: 23be movs r3, #190 ; 0xbe\n", - " c4: 23c0 movs r3, #192 ; 0xc0\n", - " c6: 23c2 movs r3, #194 ; 0xc2\n", - " c8: 23c4 movs r3, #196 ; 0xc4\n", - " ca: 23c6 movs r3, #198 ; 0xc6\n", - " cc: 23c8 movs r3, #200 ; 0xc8\n", - " ce: 23ca movs r3, #202 ; 0xca\n", - " d0: 23cc movs r3, #204 ; 0xcc\n", - " d2: 23ce movs r3, #206 ; 0xce\n", - " d4: 23d0 movs r3, #208 ; 0xd0\n", - " d6: 23d2 movs r3, #210 ; 0xd2\n", - " d8: 23d4 movs r3, #212 ; 0xd4\n", - " da: 23d6 movs r3, #214 ; 0xd6\n", - " dc: 23d8 movs r3, #216 ; 0xd8\n", - " de: 23da movs r3, #218 ; 0xda\n", - " e0: 23dc movs r3, #220 ; 0xdc\n", - " e2: 23de movs r3, #222 ; 0xde\n", - " e4: 23e0 movs r3, #224 ; 0xe0\n", - " e6: 23e2 movs r3, #226 ; 0xe2\n", - " e8: 23e4 movs r3, #228 ; 0xe4\n", - " ea: 23e6 movs r3, #230 ; 0xe6\n", - " ec: 23e8 movs r3, #232 ; 0xe8\n", - " ee: 23ea movs r3, #234 ; 0xea\n", - " f0: 23ec movs r3, #236 ; 0xec\n", - " f2: 23ee movs r3, #238 ; 0xee\n", - " f4: 23f0 movs r3, #240 ; 0xf0\n", - " f6: 23f2 movs r3, #242 ; 0xf2\n", - " f8: 23f4 movs r3, #244 ; 0xf4\n", - " fa: 23f6 movs r3, #246 ; 0xf6\n", - " fc: 23f8 movs r3, #248 ; 0xf8\n", - " fe: 23fa movs r3, #250 ; 0xfa\n", - " 100: 23fc movs r3, #252 ; 0xfc\n", - " 102: 23fe movs r3, #254 ; 0xfe\n", - " 104: 2300 movs r3, #0\n", - " 106: 2302 movs r3, #2\n", - " 108: 2304 movs r3, #4\n", - " 10a: 2306 movs r3, #6\n", - " 10c: 2308 movs r3, #8\n", - " 10e: 230a movs r3, #10\n", - " 110: 230c movs r3, #12\n", - " 112: 230e movs r3, #14\n", - " 114: 2310 movs r3, #16\n", - " 116: 2312 movs r3, #18\n", - " 118: 2314 movs r3, #20\n", - " 11a: 2316 movs r3, #22\n", - " 11c: 2318 movs r3, #24\n", - " 11e: 231a movs r3, #26\n", - " 120: 231c movs r3, #28\n", - " 122: 231e movs r3, #30\n", - " 124: 2320 movs r3, #32\n", - " 126: 2322 movs r3, #34 ; 0x22\n", - " 128: 2324 movs r3, #36 ; 0x24\n", - " 12a: 2326 movs r3, #38 ; 0x26\n", - " 12c: 2328 movs r3, #40 ; 0x28\n", - " 12e: 232a movs r3, #42 ; 0x2a\n", - " 130: 232c movs r3, #44 ; 0x2c\n", - " 132: 232e movs r3, #46 ; 0x2e\n", - " 134: 2330 movs r3, #48 ; 0x30\n", - " 136: 2332 movs r3, #50 ; 0x32\n", - " 138: 2334 movs r3, #52 ; 0x34\n", - " 13a: 2336 movs r3, #54 ; 0x36\n", - " 13c: 2338 movs r3, #56 ; 0x38\n", - " 13e: 233a movs r3, #58 ; 0x3a\n", - " 140: 233c movs r3, #60 ; 0x3c\n", - " 142: 233e movs r3, #62 ; 0x3e\n", - " 144: 2340 movs r3, #64 ; 0x40\n", - " 146: 2342 movs r3, #66 ; 0x42\n", - " 148: 2344 movs r3, #68 ; 0x44\n", - " 14a: 2346 movs r3, #70 ; 0x46\n", - " 14c: 2348 movs r3, #72 ; 0x48\n", - " 14e: 234a movs r3, #74 ; 0x4a\n", - " 150: 234c movs r3, #76 ; 0x4c\n", - " 152: 234e movs r3, #78 ; 0x4e\n", - " 154: 2350 movs r3, #80 ; 0x50\n", - " 156: 2352 movs r3, #82 ; 0x52\n", - " 158: 2354 movs r3, #84 ; 0x54\n", - " 15a: 2356 movs r3, #86 ; 0x56\n", - " 15c: 2358 movs r3, #88 ; 0x58\n", - " 15e: 235a movs r3, #90 ; 0x5a\n", - " 160: 235c movs r3, #92 ; 0x5c\n", - " 162: 235e movs r3, #94 ; 0x5e\n", - " 164: 2360 movs r3, #96 ; 0x60\n", - " 166: 2362 movs r3, #98 ; 0x62\n", - " 168: 2364 movs r3, #100 ; 0x64\n", - " 16a: 2366 movs r3, #102 ; 0x66\n", - " 16c: 2368 movs r3, #104 ; 0x68\n", - " 16e: 236a movs r3, #106 ; 0x6a\n", - " 170: 236c movs r3, #108 ; 0x6c\n", - " 172: 236e movs r3, #110 ; 0x6e\n", - " 174: 2370 movs r3, #112 ; 0x70\n", - " 176: 2372 movs r3, #114 ; 0x72\n", - " 178: 2374 movs r3, #116 ; 0x74\n", - " 17a: 2376 movs r3, #118 ; 0x76\n", - " 17c: 2378 movs r3, #120 ; 0x78\n", - " 17e: 237a movs r3, #122 ; 0x7a\n", - " 180: 237c movs r3, #124 ; 0x7c\n", - " 182: 237e movs r3, #126 ; 0x7e\n", - " 184: 2380 movs r3, #128 ; 0x80\n", - " 186: 2382 movs r3, #130 ; 0x82\n", - " 188: 2384 movs r3, #132 ; 0x84\n", - " 18a: 2386 movs r3, #134 ; 0x86\n", - " 18c: 2388 movs r3, #136 ; 0x88\n", - " 18e: 238a movs r3, #138 ; 0x8a\n", - " 190: 238c movs r3, #140 ; 0x8c\n", - " 192: 238e movs r3, #142 ; 0x8e\n", - " 194: 2390 movs r3, #144 ; 0x90\n", - " 196: 2392 movs r3, #146 ; 0x92\n", - " 198: 2394 movs r3, #148 ; 0x94\n", - " 19a: 2396 movs r3, #150 ; 0x96\n", - " 19c: 2398 movs r3, #152 ; 0x98\n", - " 19e: 239a movs r3, #154 ; 0x9a\n", - " 1a0: 239c movs r3, #156 ; 0x9c\n", - " 1a2: 239e movs r3, #158 ; 0x9e\n", - " 1a4: 23a0 movs r3, #160 ; 0xa0\n", - " 1a6: 23a2 movs r3, #162 ; 0xa2\n", - " 1a8: 23a4 movs r3, #164 ; 0xa4\n", - " 1aa: 23a6 movs r3, #166 ; 0xa6\n", - " 1ac: 23a8 movs r3, #168 ; 0xa8\n", - " 1ae: 23aa movs r3, #170 ; 0xaa\n", - " 1b0: 23ac movs r3, #172 ; 0xac\n", - " 1b2: 23ae movs r3, #174 ; 0xae\n", - " 1b4: 23b0 movs r3, #176 ; 0xb0\n", - " 1b6: 23b2 movs r3, #178 ; 0xb2\n", - " 1b8: 23b4 movs r3, #180 ; 0xb4\n", - " 1ba: 23b6 movs r3, #182 ; 0xb6\n", - " 1bc: 23b8 movs r3, #184 ; 0xb8\n", - " 1be: 23ba movs r3, #186 ; 0xba\n", - " 1c0: 23bc movs r3, #188 ; 0xbc\n", - " 1c2: 23be movs r3, #190 ; 0xbe\n", - " 1c4: 23c0 movs r3, #192 ; 0xc0\n", - " 1c6: 23c2 movs r3, #194 ; 0xc2\n", - " 1c8: 23c4 movs r3, #196 ; 0xc4\n", - " 1ca: 23c6 movs r3, #198 ; 0xc6\n", - " 1cc: 23c8 movs r3, #200 ; 0xc8\n", - " 1ce: 23ca movs r3, #202 ; 0xca\n", - " 1d0: 23cc movs r3, #204 ; 0xcc\n", - " 1d2: 23ce movs r3, #206 ; 0xce\n", - " 1d4: 23d0 movs r3, #208 ; 0xd0\n", - " 1d6: 23d2 movs r3, #210 ; 0xd2\n", - " 1d8: 23d4 movs r3, #212 ; 0xd4\n", - " 1da: 23d6 movs r3, #214 ; 0xd6\n", - " 1dc: 23d8 movs r3, #216 ; 0xd8\n", - " 1de: 23da movs r3, #218 ; 0xda\n", - " 1e0: 23dc movs r3, #220 ; 0xdc\n", - " 1e2: 23de movs r3, #222 ; 0xde\n", - " 1e4: 23e0 movs r3, #224 ; 0xe0\n", - " 1e6: 23e2 movs r3, #226 ; 0xe2\n", - " 1e8: 23e4 movs r3, #228 ; 0xe4\n", - " 1ea: 23e6 movs r3, #230 ; 0xe6\n", - " 1ec: 23e8 movs r3, #232 ; 0xe8\n", - " 1ee: 23ea movs r3, #234 ; 0xea\n", - " 1f0: 23ec movs r3, #236 ; 0xec\n", - " 1f2: 23ee movs r3, #238 ; 0xee\n", - " 1f4: 23f0 movs r3, #240 ; 0xf0\n", - " 1f6: 23f2 movs r3, #242 ; 0xf2\n", - " 1f8: 23f4 movs r3, #244 ; 0xf4\n", - " 1fa: 23f6 movs r3, #246 ; 0xf6\n", - " 1fc: 23f8 movs r3, #248 ; 0xf8\n", - " 1fe: 23fa movs r3, #250 ; 0xfa\n", - " 200: 23fc movs r3, #252 ; 0xfc\n", - " 202: 23fe movs r3, #254 ; 0xfe\n", - " 204: 2300 movs r3, #0\n", - " 206: 2302 movs r3, #2\n", - " 208: 2304 movs r3, #4\n", - " 20a: 2306 movs r3, #6\n", - " 20c: 2308 movs r3, #8\n", - " 20e: 230a movs r3, #10\n", - " 210: 230c movs r3, #12\n", - " 212: 230e movs r3, #14\n", - " 214: 2310 movs r3, #16\n", - " 216: 2312 movs r3, #18\n", - " 218: 2314 movs r3, #20\n", - " 21a: 2316 movs r3, #22\n", - " 21c: 2318 movs r3, #24\n", - " 21e: 231a movs r3, #26\n", - " 220: 231c movs r3, #28\n", - " 222: 231e movs r3, #30\n", - " 224: 2320 movs r3, #32\n", - " 226: 2322 movs r3, #34 ; 0x22\n", - " 228: 2324 movs r3, #36 ; 0x24\n", - " 22a: 2326 movs r3, #38 ; 0x26\n", - " 22c: 2328 movs r3, #40 ; 0x28\n", - " 22e: 232a movs r3, #42 ; 0x2a\n", - " 230: 232c movs r3, #44 ; 0x2c\n", - " 232: 232e movs r3, #46 ; 0x2e\n", - " 234: 2330 movs r3, #48 ; 0x30\n", - " 236: 2332 movs r3, #50 ; 0x32\n", - " 238: 2334 movs r3, #52 ; 0x34\n", - " 23a: 2336 movs r3, #54 ; 0x36\n", - " 23c: 2338 movs r3, #56 ; 0x38\n", - " 23e: 233a movs r3, #58 ; 0x3a\n", - " 240: 233c movs r3, #60 ; 0x3c\n", - " 242: 233e movs r3, #62 ; 0x3e\n", - " 244: 2340 movs r3, #64 ; 0x40\n", - " 246: 2342 movs r3, #66 ; 0x42\n", - " 248: 2344 movs r3, #68 ; 0x44\n", - " 24a: 2346 movs r3, #70 ; 0x46\n", - " 24c: 2348 movs r3, #72 ; 0x48\n", - " 24e: 234a movs r3, #74 ; 0x4a\n", - " 250: 234c movs r3, #76 ; 0x4c\n", - " 252: 234e movs r3, #78 ; 0x4e\n", - " 254: 2350 movs r3, #80 ; 0x50\n", - " 256: 2352 movs r3, #82 ; 0x52\n", - " 258: 2354 movs r3, #84 ; 0x54\n", - " 25a: 2356 movs r3, #86 ; 0x56\n", - " 25c: 2358 movs r3, #88 ; 0x58\n", - " 25e: 235a movs r3, #90 ; 0x5a\n", - " 260: 235c movs r3, #92 ; 0x5c\n", - " 262: 235e movs r3, #94 ; 0x5e\n", - " 264: 2360 movs r3, #96 ; 0x60\n", - " 266: 2362 movs r3, #98 ; 0x62\n", - " 268: 2364 movs r3, #100 ; 0x64\n", - " 26a: 2366 movs r3, #102 ; 0x66\n", - " 26c: 2368 movs r3, #104 ; 0x68\n", - " 26e: 236a movs r3, #106 ; 0x6a\n", - " 270: 236c movs r3, #108 ; 0x6c\n", - " 272: 236e movs r3, #110 ; 0x6e\n", - " 274: 2370 movs r3, #112 ; 0x70\n", - " 276: 2372 movs r3, #114 ; 0x72\n", - " 278: 2374 movs r3, #116 ; 0x74\n", - " 27a: 2376 movs r3, #118 ; 0x76\n", - " 27c: 2378 movs r3, #120 ; 0x78\n", - " 27e: 237a movs r3, #122 ; 0x7a\n", - " 280: 237c movs r3, #124 ; 0x7c\n", - " 282: 237e movs r3, #126 ; 0x7e\n", - " 284: 2380 movs r3, #128 ; 0x80\n", - " 286: 2382 movs r3, #130 ; 0x82\n", - " 288: 2384 movs r3, #132 ; 0x84\n", - " 28a: 2386 movs r3, #134 ; 0x86\n", - " 28c: 2388 movs r3, #136 ; 0x88\n", - " 28e: 238a movs r3, #138 ; 0x8a\n", - " 290: 238c movs r3, #140 ; 0x8c\n", - " 292: 238e movs r3, #142 ; 0x8e\n", - " 294: 2390 movs r3, #144 ; 0x90\n", - " 296: 2392 movs r3, #146 ; 0x92\n", - " 298: 2394 movs r3, #148 ; 0x94\n", - " 29a: 2396 movs r3, #150 ; 0x96\n", - " 29c: 2398 movs r3, #152 ; 0x98\n", - " 29e: 239a movs r3, #154 ; 0x9a\n", - " 2a0: 239c movs r3, #156 ; 0x9c\n", - " 2a2: 239e movs r3, #158 ; 0x9e\n", - " 2a4: 23a0 movs r3, #160 ; 0xa0\n", - " 2a6: 23a2 movs r3, #162 ; 0xa2\n", - " 2a8: 23a4 movs r3, #164 ; 0xa4\n", - " 2aa: 23a6 movs r3, #166 ; 0xa6\n", - " 2ac: 23a8 movs r3, #168 ; 0xa8\n", - " 2ae: 23aa movs r3, #170 ; 0xaa\n", - " 2b0: 23ac movs r3, #172 ; 0xac\n", - " 2b2: 23ae movs r3, #174 ; 0xae\n", - " 2b4: 23b0 movs r3, #176 ; 0xb0\n", - " 2b6: 23b2 movs r3, #178 ; 0xb2\n", - " 2b8: 23b4 movs r3, #180 ; 0xb4\n", - " 2ba: 23b6 movs r3, #182 ; 0xb6\n", - " 2bc: 23b8 movs r3, #184 ; 0xb8\n", - " 2be: 23ba movs r3, #186 ; 0xba\n", - " 2c0: 23bc movs r3, #188 ; 0xbc\n", - " 2c2: 23be movs r3, #190 ; 0xbe\n", - " 2c4: 23c0 movs r3, #192 ; 0xc0\n", - " 2c6: 23c2 movs r3, #194 ; 0xc2\n", - " 2c8: 23c4 movs r3, #196 ; 0xc4\n", - " 2ca: 23c6 movs r3, #198 ; 0xc6\n", - " 2cc: 23c8 movs r3, #200 ; 0xc8\n", - " 2ce: 23ca movs r3, #202 ; 0xca\n", - " 2d0: 23cc movs r3, #204 ; 0xcc\n", - " 2d2: 23ce movs r3, #206 ; 0xce\n", - " 2d4: 23d0 movs r3, #208 ; 0xd0\n", - " 2d6: 23d2 movs r3, #210 ; 0xd2\n", - " 2d8: 23d4 movs r3, #212 ; 0xd4\n", - " 2da: 23d6 movs r3, #214 ; 0xd6\n", - " 2dc: 23d8 movs r3, #216 ; 0xd8\n", - " 2de: 23da movs r3, #218 ; 0xda\n", - " 2e0: 23dc movs r3, #220 ; 0xdc\n", - " 2e2: 23de movs r3, #222 ; 0xde\n", - " 2e4: 23e0 movs r3, #224 ; 0xe0\n", - " 2e6: 23e2 movs r3, #226 ; 0xe2\n", - " 2e8: 23e4 movs r3, #228 ; 0xe4\n", - " 2ea: 23e6 movs r3, #230 ; 0xe6\n", - " 2ec: 23e8 movs r3, #232 ; 0xe8\n", - " 2ee: 23ea movs r3, #234 ; 0xea\n", - " 2f0: 23ec movs r3, #236 ; 0xec\n", - " 2f2: 23ee movs r3, #238 ; 0xee\n", - " 2f4: 23f0 movs r3, #240 ; 0xf0\n", - " 2f6: 23f2 movs r3, #242 ; 0xf2\n", - " 2f8: 23f4 movs r3, #244 ; 0xf4\n", - " 2fa: 23f6 movs r3, #246 ; 0xf6\n", - " 2fc: 23f8 movs r3, #248 ; 0xf8\n", - " 2fe: 23fa movs r3, #250 ; 0xfa\n", - " 300: 23fc movs r3, #252 ; 0xfc\n", - " 302: 23fe movs r3, #254 ; 0xfe\n", - " 304: 2300 movs r3, #0\n", - " 306: 2302 movs r3, #2\n", - " 308: 2304 movs r3, #4\n", - " 30a: 2306 movs r3, #6\n", - " 30c: 2308 movs r3, #8\n", - " 30e: 230a movs r3, #10\n", - " 310: 230c movs r3, #12\n", - " 312: 230e movs r3, #14\n", - " 314: 2310 movs r3, #16\n", - " 316: 2312 movs r3, #18\n", - " 318: 2314 movs r3, #20\n", - " 31a: 2316 movs r3, #22\n", - " 31c: 2318 movs r3, #24\n", - " 31e: 231a movs r3, #26\n", - " 320: 231c movs r3, #28\n", - " 322: 231e movs r3, #30\n", - " 324: 2320 movs r3, #32\n", - " 326: 2322 movs r3, #34 ; 0x22\n", - " 328: 2324 movs r3, #36 ; 0x24\n", - " 32a: 2326 movs r3, #38 ; 0x26\n", - " 32c: 2328 movs r3, #40 ; 0x28\n", - " 32e: 232a movs r3, #42 ; 0x2a\n", - " 330: 232c movs r3, #44 ; 0x2c\n", - " 332: 232e movs r3, #46 ; 0x2e\n", - " 334: 2330 movs r3, #48 ; 0x30\n", - " 336: 2332 movs r3, #50 ; 0x32\n", - " 338: 2334 movs r3, #52 ; 0x34\n", - " 33a: 2336 movs r3, #54 ; 0x36\n", - " 33c: 2338 movs r3, #56 ; 0x38\n", - " 33e: 233a movs r3, #58 ; 0x3a\n", - " 340: 233c movs r3, #60 ; 0x3c\n", - " 342: 233e movs r3, #62 ; 0x3e\n", - " 344: 2340 movs r3, #64 ; 0x40\n", - " 346: 2342 movs r3, #66 ; 0x42\n", - " 348: 2344 movs r3, #68 ; 0x44\n", - " 34a: 2346 movs r3, #70 ; 0x46\n", - " 34c: 2348 movs r3, #72 ; 0x48\n", - " 34e: 234a movs r3, #74 ; 0x4a\n", - " 350: 234c movs r3, #76 ; 0x4c\n", - " 352: 234e movs r3, #78 ; 0x4e\n", - " 354: 2350 movs r3, #80 ; 0x50\n", - " 356: 2352 movs r3, #82 ; 0x52\n", - " 358: 2354 movs r3, #84 ; 0x54\n", - " 35a: 2356 movs r3, #86 ; 0x56\n", - " 35c: 2358 movs r3, #88 ; 0x58\n", - " 35e: 235a movs r3, #90 ; 0x5a\n", - " 360: 235c movs r3, #92 ; 0x5c\n", - " 362: 235e movs r3, #94 ; 0x5e\n", - " 364: 2360 movs r3, #96 ; 0x60\n", - " 366: 2362 movs r3, #98 ; 0x62\n", - " 368: 2364 movs r3, #100 ; 0x64\n", - " 36a: 2366 movs r3, #102 ; 0x66\n", - " 36c: 2368 movs r3, #104 ; 0x68\n", - " 36e: 236a movs r3, #106 ; 0x6a\n", - " 370: 236c movs r3, #108 ; 0x6c\n", - " 372: 236e movs r3, #110 ; 0x6e\n", - " 374: 2370 movs r3, #112 ; 0x70\n", - " 376: 2372 movs r3, #114 ; 0x72\n", - " 378: 2374 movs r3, #116 ; 0x74\n", - " 37a: 2376 movs r3, #118 ; 0x76\n", - " 37c: 2378 movs r3, #120 ; 0x78\n", - " 37e: 237a movs r3, #122 ; 0x7a\n", - " 380: 237c movs r3, #124 ; 0x7c\n", - " 382: 237e movs r3, #126 ; 0x7e\n", - " 384: 2380 movs r3, #128 ; 0x80\n", - " 386: 2382 movs r3, #130 ; 0x82\n", - " 388: 2384 movs r3, #132 ; 0x84\n", - " 38a: 2386 movs r3, #134 ; 0x86\n", - " 38c: 2388 movs r3, #136 ; 0x88\n", - " 38e: 238a movs r3, #138 ; 0x8a\n", - " 390: 238c movs r3, #140 ; 0x8c\n", - " 392: 238e movs r3, #142 ; 0x8e\n", - " 394: 2390 movs r3, #144 ; 0x90\n", - " 396: 2392 movs r3, #146 ; 0x92\n", - " 398: 2394 movs r3, #148 ; 0x94\n", - " 39a: 2396 movs r3, #150 ; 0x96\n", - " 39c: 2398 movs r3, #152 ; 0x98\n", - " 39e: 239a movs r3, #154 ; 0x9a\n", - " 3a0: 239c movs r3, #156 ; 0x9c\n", - " 3a2: 239e movs r3, #158 ; 0x9e\n", - " 3a4: 23a0 movs r3, #160 ; 0xa0\n", - " 3a6: 23a2 movs r3, #162 ; 0xa2\n", - " 3a8: 23a4 movs r3, #164 ; 0xa4\n", - " 3aa: 23a6 movs r3, #166 ; 0xa6\n", - " 3ac: 23a8 movs r3, #168 ; 0xa8\n", - " 3ae: 23aa movs r3, #170 ; 0xaa\n", - " 3b0: 23ac movs r3, #172 ; 0xac\n", - " 3b2: 23ae movs r3, #174 ; 0xae\n", - " 3b4: 23b0 movs r3, #176 ; 0xb0\n", - " 3b6: 23b2 movs r3, #178 ; 0xb2\n", - " 3b8: 23b4 movs r3, #180 ; 0xb4\n", - " 3ba: 23b6 movs r3, #182 ; 0xb6\n", - " 3bc: 23b8 movs r3, #184 ; 0xb8\n", - " 3be: 23ba movs r3, #186 ; 0xba\n", - " 3c0: 23bc movs r3, #188 ; 0xbc\n", - " 3c2: 23be movs r3, #190 ; 0xbe\n", - " 3c4: 23c0 movs r3, #192 ; 0xc0\n", - " 3c6: 23c2 movs r3, #194 ; 0xc2\n", - " 3c8: 23c4 movs r3, #196 ; 0xc4\n", - " 3ca: 23c6 movs r3, #198 ; 0xc6\n", - " 3cc: 23c8 movs r3, #200 ; 0xc8\n", - " 3ce: 23ca movs r3, #202 ; 0xca\n", - " 3d0: 23cc movs r3, #204 ; 0xcc\n", - " 3d2: 23ce movs r3, #206 ; 0xce\n", - " 3d4: 23d0 movs r3, #208 ; 0xd0\n", - " 3d6: 23d2 movs r3, #210 ; 0xd2\n", - " 3d8: 23d4 movs r3, #212 ; 0xd4\n", - " 3da: 23d6 movs r3, #214 ; 0xd6\n", - " 3dc: 23d8 movs r3, #216 ; 0xd8\n", - " 3de: 23da movs r3, #218 ; 0xda\n", - " 3e0: 23dc movs r3, #220 ; 0xdc\n", - " 3e2: 23de movs r3, #222 ; 0xde\n", - " 3e4: 23e0 movs r3, #224 ; 0xe0\n", - " 3e6: 23e2 movs r3, #226 ; 0xe2\n", - " 3e8: 23e4 movs r3, #228 ; 0xe4\n", - " 3ea: 23e6 movs r3, #230 ; 0xe6\n", - " 3ec: 23e8 movs r3, #232 ; 0xe8\n", - " 3ee: 23ea movs r3, #234 ; 0xea\n", - " 3f0: 23ec movs r3, #236 ; 0xec\n", - " 3f2: 23ee movs r3, #238 ; 0xee\n", - " 3f4: 23f0 movs r3, #240 ; 0xf0\n", - " 3f6: 23f2 movs r3, #242 ; 0xf2\n", - " 3f8: 23f4 movs r3, #244 ; 0xf4\n", - " 3fa: 23f6 movs r3, #246 ; 0xf6\n", - " 3fc: 23f8 movs r3, #248 ; 0xf8\n", - " 3fe: 23fa movs r3, #250 ; 0xfa\n", - " 400: 23fc movs r3, #252 ; 0xfc\n", - " 402: 23fe movs r3, #254 ; 0xfe\n", - " 404: 2300 movs r3, #0\n", - " 406: 2302 movs r3, #2\n", - " 408: 2304 movs r3, #4\n", - " 40a: 2306 movs r3, #6\n", - " 40c: 2308 movs r3, #8\n", - " 40e: 230a movs r3, #10\n", - " 410: 230c movs r3, #12\n", - " 412: 230e movs r3, #14\n", - " 414: 2310 movs r3, #16\n", - " 416: 2312 movs r3, #18\n", - " 418: 2314 movs r3, #20\n", - " 41a: 2316 movs r3, #22\n", - " 41c: 2318 movs r3, #24\n", - " 41e: 231a movs r3, #26\n", - " 420: 231c movs r3, #28\n", - " 422: 231e movs r3, #30\n", - " 424: 2320 movs r3, #32\n", - " 426: 2322 movs r3, #34 ; 0x22\n", - " 428: 2324 movs r3, #36 ; 0x24\n", - " 42a: 2326 movs r3, #38 ; 0x26\n", - " 42c: 2328 movs r3, #40 ; 0x28\n", - " 42e: 232a movs r3, #42 ; 0x2a\n", - " 430: 232c movs r3, #44 ; 0x2c\n", - " 432: 232e movs r3, #46 ; 0x2e\n", - " 434: 2330 movs r3, #48 ; 0x30\n", - " 436: 2332 movs r3, #50 ; 0x32\n", - " 438: 2334 movs r3, #52 ; 0x34\n", - " 43a: 2336 movs r3, #54 ; 0x36\n", - " 43c: 2338 movs r3, #56 ; 0x38\n", - " 43e: 233a movs r3, #58 ; 0x3a\n", - " 440: 233c movs r3, #60 ; 0x3c\n", - " 442: 233e movs r3, #62 ; 0x3e\n", - " 444: 2340 movs r3, #64 ; 0x40\n", - " 446: 2342 movs r3, #66 ; 0x42\n", - " 448: 2344 movs r3, #68 ; 0x44\n", - " 44a: 2346 movs r3, #70 ; 0x46\n", - " 44c: 2348 movs r3, #72 ; 0x48\n", - " 44e: 234a movs r3, #74 ; 0x4a\n", - " 450: 234c movs r3, #76 ; 0x4c\n", - " 452: 234e movs r3, #78 ; 0x4e\n", - " 454: 2350 movs r3, #80 ; 0x50\n", - " 456: 2352 movs r3, #82 ; 0x52\n", - " 458: 2354 movs r3, #84 ; 0x54\n", - " 45a: 2356 movs r3, #86 ; 0x56\n", - " 45c: 2358 movs r3, #88 ; 0x58\n", - " 45e: 235a movs r3, #90 ; 0x5a\n", - " 460: 235c movs r3, #92 ; 0x5c\n", - " 462: 235e movs r3, #94 ; 0x5e\n", - " 464: 2360 movs r3, #96 ; 0x60\n", - " 466: 2362 movs r3, #98 ; 0x62\n", - " 468: 2364 movs r3, #100 ; 0x64\n", - " 46a: 2366 movs r3, #102 ; 0x66\n", - " 46c: 2368 movs r3, #104 ; 0x68\n", - " 46e: 236a movs r3, #106 ; 0x6a\n", - " 470: 236c movs r3, #108 ; 0x6c\n", - " 472: 236e movs r3, #110 ; 0x6e\n", - " 474: 2370 movs r3, #112 ; 0x70\n", - " 476: 2372 movs r3, #114 ; 0x72\n", - " 478: 2374 movs r3, #116 ; 0x74\n", - " 47a: 2376 movs r3, #118 ; 0x76\n", - " 47c: 2378 movs r3, #120 ; 0x78\n", - " 47e: 237a movs r3, #122 ; 0x7a\n", - " 480: 237c movs r3, #124 ; 0x7c\n", - " 482: 237e movs r3, #126 ; 0x7e\n", - " 484: 2380 movs r3, #128 ; 0x80\n", - " 486: 2382 movs r3, #130 ; 0x82\n", - " 488: 2384 movs r3, #132 ; 0x84\n", - " 48a: 2386 movs r3, #134 ; 0x86\n", - " 48c: 2388 movs r3, #136 ; 0x88\n", - " 48e: 238a movs r3, #138 ; 0x8a\n", - " 490: 238c movs r3, #140 ; 0x8c\n", - " 492: 238e movs r3, #142 ; 0x8e\n", - " 494: 2390 movs r3, #144 ; 0x90\n", - " 496: 2392 movs r3, #146 ; 0x92\n", - " 498: 2394 movs r3, #148 ; 0x94\n", - " 49a: 2396 movs r3, #150 ; 0x96\n", - " 49c: 2398 movs r3, #152 ; 0x98\n", - " 49e: 239a movs r3, #154 ; 0x9a\n", - " 4a0: 239c movs r3, #156 ; 0x9c\n", - " 4a2: 239e movs r3, #158 ; 0x9e\n", - " 4a4: 23a0 movs r3, #160 ; 0xa0\n", - " 4a6: 23a2 movs r3, #162 ; 0xa2\n", - " 4a8: 23a4 movs r3, #164 ; 0xa4\n", - " 4aa: 23a6 movs r3, #166 ; 0xa6\n", - " 4ac: 23a8 movs r3, #168 ; 0xa8\n", - " 4ae: 23aa movs r3, #170 ; 0xaa\n", - " 4b0: 23ac movs r3, #172 ; 0xac\n", - " 4b2: 23ae movs r3, #174 ; 0xae\n", - " 4b4: 23b0 movs r3, #176 ; 0xb0\n", - " 4b6: 23b2 movs r3, #178 ; 0xb2\n", - " 4b8: 23b4 movs r3, #180 ; 0xb4\n", - " 4ba: 23b6 movs r3, #182 ; 0xb6\n", - " 4bc: 23b8 movs r3, #184 ; 0xb8\n", - " 4be: 23ba movs r3, #186 ; 0xba\n", - " 4c0: 23bc movs r3, #188 ; 0xbc\n", - " 4c2: 23be movs r3, #190 ; 0xbe\n", - " 4c4: 23c0 movs r3, #192 ; 0xc0\n", - " 4c6: 23c2 movs r3, #194 ; 0xc2\n", - " 4c8: 23c4 movs r3, #196 ; 0xc4\n", - " 4ca: 23c6 movs r3, #198 ; 0xc6\n", - " 4cc: 23c8 movs r3, #200 ; 0xc8\n", - " 4ce: 23ca movs r3, #202 ; 0xca\n", - " 4d0: 23cc movs r3, #204 ; 0xcc\n", - " 4d2: 23ce movs r3, #206 ; 0xce\n", - " 4d4: 23d0 movs r3, #208 ; 0xd0\n", - " 4d6: 23d2 movs r3, #210 ; 0xd2\n", - " 4d8: 23d4 movs r3, #212 ; 0xd4\n", - " 4da: 23d6 movs r3, #214 ; 0xd6\n", - " 4dc: 23d8 movs r3, #216 ; 0xd8\n", - " 4de: 23da movs r3, #218 ; 0xda\n", - " 4e0: 23dc movs r3, #220 ; 0xdc\n", - " 4e2: 23de movs r3, #222 ; 0xde\n", - " 4e4: 23e0 movs r3, #224 ; 0xe0\n", - " 4e6: 23e2 movs r3, #226 ; 0xe2\n", - " 4e8: 23e4 movs r3, #228 ; 0xe4\n", - " 4ea: 23e6 movs r3, #230 ; 0xe6\n", - " 4ec: 23e8 movs r3, #232 ; 0xe8\n", - " 4ee: 23ea movs r3, #234 ; 0xea\n", - " 4f0: 23ec movs r3, #236 ; 0xec\n", - " 4f2: 23ee movs r3, #238 ; 0xee\n", - " 4f4: 23f0 movs r3, #240 ; 0xf0\n", - " 4f6: 23f2 movs r3, #242 ; 0xf2\n", - " 4f8: 23f4 movs r3, #244 ; 0xf4\n", - " 4fa: 23f6 movs r3, #246 ; 0xf6\n", - " 4fc: 23f8 movs r3, #248 ; 0xf8\n", - " 4fe: 23fa movs r3, #250 ; 0xfa\n", - " 500: 23fc movs r3, #252 ; 0xfc\n", - " 502: 23fe movs r3, #254 ; 0xfe\n", - " 504: 2300 movs r3, #0\n", - " 506: 2302 movs r3, #2\n", - " 508: 2304 movs r3, #4\n", - " 50a: 2306 movs r3, #6\n", - " 50c: 2308 movs r3, #8\n", - " 50e: 230a movs r3, #10\n", - " 510: 230c movs r3, #12\n", - " 512: 230e movs r3, #14\n", - " 514: 2310 movs r3, #16\n", - " 516: 2312 movs r3, #18\n", - " 518: 2314 movs r3, #20\n", - " 51a: 2316 movs r3, #22\n", - " 51c: 2318 movs r3, #24\n", - " 51e: 231a movs r3, #26\n", - " 520: 231c movs r3, #28\n", - " 522: 231e movs r3, #30\n", - " 524: 2320 movs r3, #32\n", - " 526: 2322 movs r3, #34 ; 0x22\n", - " 528: 2324 movs r3, #36 ; 0x24\n", - " 52a: 2326 movs r3, #38 ; 0x26\n", - " 52c: 2328 movs r3, #40 ; 0x28\n", - " 52e: 232a movs r3, #42 ; 0x2a\n", - " 530: 232c movs r3, #44 ; 0x2c\n", - " 532: 232e movs r3, #46 ; 0x2e\n", - " 534: 2330 movs r3, #48 ; 0x30\n", - " 536: 2332 movs r3, #50 ; 0x32\n", - " 538: 2334 movs r3, #52 ; 0x34\n", - " 53a: 2336 movs r3, #54 ; 0x36\n", - " 53c: 2338 movs r3, #56 ; 0x38\n", - " 53e: 233a movs r3, #58 ; 0x3a\n", - " 540: 233c movs r3, #60 ; 0x3c\n", - " 542: 233e movs r3, #62 ; 0x3e\n", - " 544: 2340 movs r3, #64 ; 0x40\n", - " 546: 2342 movs r3, #66 ; 0x42\n", - " 548: 2344 movs r3, #68 ; 0x44\n", - " 54a: 2346 movs r3, #70 ; 0x46\n", - " 54c: 2348 movs r3, #72 ; 0x48\n", - " 54e: 234a movs r3, #74 ; 0x4a\n", - " 550: 234c movs r3, #76 ; 0x4c\n", - " 552: 234e movs r3, #78 ; 0x4e\n", - " 554: 2350 movs r3, #80 ; 0x50\n", - " 556: 2352 movs r3, #82 ; 0x52\n", - " 558: 2354 movs r3, #84 ; 0x54\n", - " 55a: 2356 movs r3, #86 ; 0x56\n", - " 55c: 2358 movs r3, #88 ; 0x58\n", - " 55e: 235a movs r3, #90 ; 0x5a\n", - " 560: 235c movs r3, #92 ; 0x5c\n", - " 562: 235e movs r3, #94 ; 0x5e\n", - " 564: 2360 movs r3, #96 ; 0x60\n", - " 566: 2362 movs r3, #98 ; 0x62\n", - " 568: 2364 movs r3, #100 ; 0x64\n", - " 56a: 2366 movs r3, #102 ; 0x66\n", - " 56c: 2368 movs r3, #104 ; 0x68\n", - " 56e: 236a movs r3, #106 ; 0x6a\n", - " 570: 236c movs r3, #108 ; 0x6c\n", - " 572: 236e movs r3, #110 ; 0x6e\n", - " 574: 2370 movs r3, #112 ; 0x70\n", - " 576: 2372 movs r3, #114 ; 0x72\n", - " 578: 2374 movs r3, #116 ; 0x74\n", - " 57a: 2376 movs r3, #118 ; 0x76\n", - " 57c: 2378 movs r3, #120 ; 0x78\n", - " 57e: 237a movs r3, #122 ; 0x7a\n", - " 580: 237c movs r3, #124 ; 0x7c\n", - " 582: 237e movs r3, #126 ; 0x7e\n", - " 584: 2380 movs r3, #128 ; 0x80\n", - " 586: 2382 movs r3, #130 ; 0x82\n", - " 588: 2384 movs r3, #132 ; 0x84\n", - " 58a: 2386 movs r3, #134 ; 0x86\n", - " 58c: 2388 movs r3, #136 ; 0x88\n", - " 58e: 238a movs r3, #138 ; 0x8a\n", - " 590: 238c movs r3, #140 ; 0x8c\n", - " 592: 238e movs r3, #142 ; 0x8e\n", - " 594: 2390 movs r3, #144 ; 0x90\n", - " 596: 2392 movs r3, #146 ; 0x92\n", - " 598: 2394 movs r3, #148 ; 0x94\n", - " 59a: 2396 movs r3, #150 ; 0x96\n", - " 59c: 2398 movs r3, #152 ; 0x98\n", - " 59e: 239a movs r3, #154 ; 0x9a\n", - " 5a0: 239c movs r3, #156 ; 0x9c\n", - " 5a2: 239e movs r3, #158 ; 0x9e\n", - " 5a4: 23a0 movs r3, #160 ; 0xa0\n", - " 5a6: 23a2 movs r3, #162 ; 0xa2\n", - " 5a8: 23a4 movs r3, #164 ; 0xa4\n", - " 5aa: 23a6 movs r3, #166 ; 0xa6\n", - " 5ac: 23a8 movs r3, #168 ; 0xa8\n", - " 5ae: 23aa movs r3, #170 ; 0xaa\n", - " 5b0: 23ac movs r3, #172 ; 0xac\n", - " 5b2: 23ae movs r3, #174 ; 0xae\n", - " 5b4: 23b0 movs r3, #176 ; 0xb0\n", - " 5b6: 23b2 movs r3, #178 ; 0xb2\n", - " 5b8: 23b4 movs r3, #180 ; 0xb4\n", - " 5ba: 23b6 movs r3, #182 ; 0xb6\n", - " 5bc: 23b8 movs r3, #184 ; 0xb8\n", - " 5be: 23ba movs r3, #186 ; 0xba\n", - " 5c0: 23bc movs r3, #188 ; 0xbc\n", - " 5c2: 23be movs r3, #190 ; 0xbe\n", - " 5c4: 23c0 movs r3, #192 ; 0xc0\n", - " 5c6: 23c2 movs r3, #194 ; 0xc2\n", - " 5c8: 23c4 movs r3, #196 ; 0xc4\n", - " 5ca: 23c6 movs r3, #198 ; 0xc6\n", - " 5cc: 23c8 movs r3, #200 ; 0xc8\n", - " 5ce: 23ca movs r3, #202 ; 0xca\n", - " 5d0: 23cc movs r3, #204 ; 0xcc\n", - " 5d2: 23ce movs r3, #206 ; 0xce\n", - " 5d4: 23d0 movs r3, #208 ; 0xd0\n", - " 5d6: 23d2 movs r3, #210 ; 0xd2\n", - " 5d8: 23d4 movs r3, #212 ; 0xd4\n", - " 5da: 23d6 movs r3, #214 ; 0xd6\n", - " 5dc: 23d8 movs r3, #216 ; 0xd8\n", - " 5de: 23da movs r3, #218 ; 0xda\n", - " 5e0: 23dc movs r3, #220 ; 0xdc\n", - " 5e2: 23de movs r3, #222 ; 0xde\n", - " 5e4: 23e0 movs r3, #224 ; 0xe0\n", - " 5e6: 23e2 movs r3, #226 ; 0xe2\n", - " 5e8: 23e4 movs r3, #228 ; 0xe4\n", - " 5ea: 23e6 movs r3, #230 ; 0xe6\n", - " 5ec: 23e8 movs r3, #232 ; 0xe8\n", - " 5ee: 23ea movs r3, #234 ; 0xea\n", - " 5f0: 23ec movs r3, #236 ; 0xec\n", - " 5f2: 23ee movs r3, #238 ; 0xee\n", - " 5f4: 23f0 movs r3, #240 ; 0xf0\n", - " 5f6: 23f2 movs r3, #242 ; 0xf2\n", - " 5f8: 23f4 movs r3, #244 ; 0xf4\n", - " 5fa: 23f6 movs r3, #246 ; 0xf6\n", - " 5fc: 23f8 movs r3, #248 ; 0xf8\n", - " 5fe: 23fa movs r3, #250 ; 0xfa\n", - " 600: 23fc movs r3, #252 ; 0xfc\n", - " 602: 23fe movs r3, #254 ; 0xfe\n", - " 604: 2300 movs r3, #0\n", - " 606: 2302 movs r3, #2\n", - " 608: 2304 movs r3, #4\n", - " 60a: 2306 movs r3, #6\n", - " 60c: 2308 movs r3, #8\n", - " 60e: 230a movs r3, #10\n", - " 610: 230c movs r3, #12\n", - " 612: 230e movs r3, #14\n", - " 614: 2310 movs r3, #16\n", - " 616: 2312 movs r3, #18\n", - " 618: 2314 movs r3, #20\n", - " 61a: 2316 movs r3, #22\n", - " 61c: 2318 movs r3, #24\n", - " 61e: 231a movs r3, #26\n", - " 620: 231c movs r3, #28\n", - " 622: 231e movs r3, #30\n", - " 624: 2320 movs r3, #32\n", - " 626: 2322 movs r3, #34 ; 0x22\n", - " 628: 2324 movs r3, #36 ; 0x24\n", - " 62a: 2326 movs r3, #38 ; 0x26\n", - " 62c: 2328 movs r3, #40 ; 0x28\n", - " 62e: 232a movs r3, #42 ; 0x2a\n", - " 630: 232c movs r3, #44 ; 0x2c\n", - " 632: 232e movs r3, #46 ; 0x2e\n", - " 634: 2330 movs r3, #48 ; 0x30\n", - " 636: 2332 movs r3, #50 ; 0x32\n", - " 638: 2334 movs r3, #52 ; 0x34\n", - " 63a: 2336 movs r3, #54 ; 0x36\n", - " 63c: 2338 movs r3, #56 ; 0x38\n", - " 63e: 233a movs r3, #58 ; 0x3a\n", - " 640: 233c movs r3, #60 ; 0x3c\n", - " 642: 233e movs r3, #62 ; 0x3e\n", - " 644: 2340 movs r3, #64 ; 0x40\n", - " 646: 2342 movs r3, #66 ; 0x42\n", - " 648: 2344 movs r3, #68 ; 0x44\n", - " 64a: 2346 movs r3, #70 ; 0x46\n", - " 64c: 2348 movs r3, #72 ; 0x48\n", - " 64e: 234a movs r3, #74 ; 0x4a\n", - " 650: 234c movs r3, #76 ; 0x4c\n", - " 652: 234e movs r3, #78 ; 0x4e\n", - " 654: 2350 movs r3, #80 ; 0x50\n", - " 656: 2352 movs r3, #82 ; 0x52\n", - " 658: 2354 movs r3, #84 ; 0x54\n", - " 65a: 2356 movs r3, #86 ; 0x56\n", - " 65c: 2358 movs r3, #88 ; 0x58\n", - " 65e: 235a movs r3, #90 ; 0x5a\n", - " 660: 235c movs r3, #92 ; 0x5c\n", - " 662: 235e movs r3, #94 ; 0x5e\n", - " 664: 2360 movs r3, #96 ; 0x60\n", - " 666: 2362 movs r3, #98 ; 0x62\n", - " 668: 2364 movs r3, #100 ; 0x64\n", - " 66a: 2366 movs r3, #102 ; 0x66\n", - " 66c: 2368 movs r3, #104 ; 0x68\n", - " 66e: 236a movs r3, #106 ; 0x6a\n", - " 670: 236c movs r3, #108 ; 0x6c\n", - " 672: 236e movs r3, #110 ; 0x6e\n", - " 674: 2370 movs r3, #112 ; 0x70\n", - " 676: 2372 movs r3, #114 ; 0x72\n", - " 678: 2374 movs r3, #116 ; 0x74\n", - " 67a: 2376 movs r3, #118 ; 0x76\n", - " 67c: 2378 movs r3, #120 ; 0x78\n", - " 67e: 237a movs r3, #122 ; 0x7a\n", - " 680: 237c movs r3, #124 ; 0x7c\n", - " 682: 237e movs r3, #126 ; 0x7e\n", - " 684: 2380 movs r3, #128 ; 0x80\n", - " 686: 2382 movs r3, #130 ; 0x82\n", - " 688: 2384 movs r3, #132 ; 0x84\n", - " 68a: 2386 movs r3, #134 ; 0x86\n", - " 68c: 2388 movs r3, #136 ; 0x88\n", - " 68e: 238a movs r3, #138 ; 0x8a\n", - " 690: 238c movs r3, #140 ; 0x8c\n", - " 692: 238e movs r3, #142 ; 0x8e\n", - " 694: 2390 movs r3, #144 ; 0x90\n", - " 696: 2392 movs r3, #146 ; 0x92\n", - " 698: 2394 movs r3, #148 ; 0x94\n", - " 69a: 2396 movs r3, #150 ; 0x96\n", - " 69c: 2398 movs r3, #152 ; 0x98\n", - " 69e: 239a movs r3, #154 ; 0x9a\n", - " 6a0: 239c movs r3, #156 ; 0x9c\n", - " 6a2: 239e movs r3, #158 ; 0x9e\n", - " 6a4: 23a0 movs r3, #160 ; 0xa0\n", - " 6a6: 23a2 movs r3, #162 ; 0xa2\n", - " 6a8: 23a4 movs r3, #164 ; 0xa4\n", - " 6aa: 23a6 movs r3, #166 ; 0xa6\n", - " 6ac: 23a8 movs r3, #168 ; 0xa8\n", - " 6ae: 23aa movs r3, #170 ; 0xaa\n", - " 6b0: 23ac movs r3, #172 ; 0xac\n", - " 6b2: 23ae movs r3, #174 ; 0xae\n", - " 6b4: 23b0 movs r3, #176 ; 0xb0\n", - " 6b6: 23b2 movs r3, #178 ; 0xb2\n", - " 6b8: 23b4 movs r3, #180 ; 0xb4\n", - " 6ba: 23b6 movs r3, #182 ; 0xb6\n", - " 6bc: 23b8 movs r3, #184 ; 0xb8\n", - " 6be: 23ba movs r3, #186 ; 0xba\n", - " 6c0: 23bc movs r3, #188 ; 0xbc\n", - " 6c2: 23be movs r3, #190 ; 0xbe\n", - " 6c4: 23c0 movs r3, #192 ; 0xc0\n", - " 6c6: 23c2 movs r3, #194 ; 0xc2\n", - " 6c8: 23c4 movs r3, #196 ; 0xc4\n", - " 6ca: 23c6 movs r3, #198 ; 0xc6\n", - " 6cc: 23c8 movs r3, #200 ; 0xc8\n", - " 6ce: 23ca movs r3, #202 ; 0xca\n", - " 6d0: 23cc movs r3, #204 ; 0xcc\n", - " 6d2: 23ce movs r3, #206 ; 0xce\n", - " 6d4: 23d0 movs r3, #208 ; 0xd0\n", - " 6d6: 23d2 movs r3, #210 ; 0xd2\n", - " 6d8: 23d4 movs r3, #212 ; 0xd4\n", - " 6da: 23d6 movs r3, #214 ; 0xd6\n", - " 6dc: 23d8 movs r3, #216 ; 0xd8\n", - " 6de: 23da movs r3, #218 ; 0xda\n", - " 6e0: 23dc movs r3, #220 ; 0xdc\n", - " 6e2: 23de movs r3, #222 ; 0xde\n", - " 6e4: 23e0 movs r3, #224 ; 0xe0\n", - " 6e6: 23e2 movs r3, #226 ; 0xe2\n", - " 6e8: 23e4 movs r3, #228 ; 0xe4\n", - " 6ea: 23e6 movs r3, #230 ; 0xe6\n", - " 6ec: 23e8 movs r3, #232 ; 0xe8\n", - " 6ee: 23ea movs r3, #234 ; 0xea\n", - " 6f0: 23ec movs r3, #236 ; 0xec\n", - " 6f2: 23ee movs r3, #238 ; 0xee\n", - " 6f4: 23f0 movs r3, #240 ; 0xf0\n", - " 6f6: 23f2 movs r3, #242 ; 0xf2\n", - " 6f8: 23f4 movs r3, #244 ; 0xf4\n", - " 6fa: 23f6 movs r3, #246 ; 0xf6\n", - " 6fc: 23f8 movs r3, #248 ; 0xf8\n", - " 6fe: 23fa movs r3, #250 ; 0xfa\n", - " 700: 23fc movs r3, #252 ; 0xfc\n", - " 702: 23fe movs r3, #254 ; 0xfe\n", - " 704: 2300 movs r3, #0\n", - " 706: 2302 movs r3, #2\n", - " 708: 2304 movs r3, #4\n", - " 70a: 2306 movs r3, #6\n", - " 70c: 2308 movs r3, #8\n", - " 70e: 230a movs r3, #10\n", - " 710: 230c movs r3, #12\n", - " 712: 230e movs r3, #14\n", - " 714: 2310 movs r3, #16\n", - " 716: 2312 movs r3, #18\n", - " 718: 2314 movs r3, #20\n", - " 71a: 2316 movs r3, #22\n", - " 71c: 2318 movs r3, #24\n", - " 71e: 231a movs r3, #26\n", - " 720: 231c movs r3, #28\n", - " 722: 231e movs r3, #30\n", - " 724: 2320 movs r3, #32\n", - " 726: 2322 movs r3, #34 ; 0x22\n", - " 728: 2324 movs r3, #36 ; 0x24\n", - " 72a: 2326 movs r3, #38 ; 0x26\n", - " 72c: 2328 movs r3, #40 ; 0x28\n", - " 72e: 232a movs r3, #42 ; 0x2a\n", - " 730: 232c movs r3, #44 ; 0x2c\n", - " 732: 232e movs r3, #46 ; 0x2e\n", - " 734: 2330 movs r3, #48 ; 0x30\n", - " 736: 2332 movs r3, #50 ; 0x32\n", - " 738: 2334 movs r3, #52 ; 0x34\n", - " 73a: 2336 movs r3, #54 ; 0x36\n", - " 73c: 2338 movs r3, #56 ; 0x38\n", - " 73e: 233a movs r3, #58 ; 0x3a\n", - " 740: 233c movs r3, #60 ; 0x3c\n", - " 742: 233e movs r3, #62 ; 0x3e\n", - " 744: 2340 movs r3, #64 ; 0x40\n", - " 746: 2342 movs r3, #66 ; 0x42\n", - " 748: 2344 movs r3, #68 ; 0x44\n", - " 74a: 2346 movs r3, #70 ; 0x46\n", - " 74c: 2348 movs r3, #72 ; 0x48\n", - " 74e: 234a movs r3, #74 ; 0x4a\n", - " 750: 234c movs r3, #76 ; 0x4c\n", - " 752: 234e movs r3, #78 ; 0x4e\n", - " 754: 2350 movs r3, #80 ; 0x50\n", - " 756: 2352 movs r3, #82 ; 0x52\n", - " 758: 2354 movs r3, #84 ; 0x54\n", - " 75a: 2356 movs r3, #86 ; 0x56\n", - " 75c: 2358 movs r3, #88 ; 0x58\n", - " 75e: 235a movs r3, #90 ; 0x5a\n", - " 760: 235c movs r3, #92 ; 0x5c\n", - " 762: 235e movs r3, #94 ; 0x5e\n", - " 764: 2360 movs r3, #96 ; 0x60\n", - " 766: 2362 movs r3, #98 ; 0x62\n", - " 768: 2364 movs r3, #100 ; 0x64\n", - " 76a: 2366 movs r3, #102 ; 0x66\n", - " 76c: 2368 movs r3, #104 ; 0x68\n", - " 76e: 236a movs r3, #106 ; 0x6a\n", - " 770: 236c movs r3, #108 ; 0x6c\n", - " 772: 236e movs r3, #110 ; 0x6e\n", - " 774: 2370 movs r3, #112 ; 0x70\n", - " 776: 2372 movs r3, #114 ; 0x72\n", - " 778: 2374 movs r3, #116 ; 0x74\n", - " 77a: 2376 movs r3, #118 ; 0x76\n", - " 77c: 2378 movs r3, #120 ; 0x78\n", - " 77e: 237a movs r3, #122 ; 0x7a\n", - " 780: 237c movs r3, #124 ; 0x7c\n", - " 782: 237e movs r3, #126 ; 0x7e\n", - " 784: 2380 movs r3, #128 ; 0x80\n", - " 786: 2382 movs r3, #130 ; 0x82\n", - " 788: 2384 movs r3, #132 ; 0x84\n", - " 78a: 2386 movs r3, #134 ; 0x86\n", - " 78c: 2388 movs r3, #136 ; 0x88\n", - " 78e: 238a movs r3, #138 ; 0x8a\n", - " 790: 238c movs r3, #140 ; 0x8c\n", - " 792: 238e movs r3, #142 ; 0x8e\n", - " 794: 2390 movs r3, #144 ; 0x90\n", - " 796: 2392 movs r3, #146 ; 0x92\n", - " 798: 2394 movs r3, #148 ; 0x94\n", - " 79a: 2396 movs r3, #150 ; 0x96\n", - " 79c: 2398 movs r3, #152 ; 0x98\n", - " 79e: 239a movs r3, #154 ; 0x9a\n", - " 7a0: 239c movs r3, #156 ; 0x9c\n", - " 7a2: 239e movs r3, #158 ; 0x9e\n", - " 7a4: 23a0 movs r3, #160 ; 0xa0\n", - " 7a6: 23a2 movs r3, #162 ; 0xa2\n", - " 7a8: 23a4 movs r3, #164 ; 0xa4\n", - " 7aa: 23a6 movs r3, #166 ; 0xa6\n", - " 7ac: 23a8 movs r3, #168 ; 0xa8\n", - " 7ae: 23aa movs r3, #170 ; 0xaa\n", - " 7b0: 23ac movs r3, #172 ; 0xac\n", - " 7b2: 23ae movs r3, #174 ; 0xae\n", - " 7b4: 23b0 movs r3, #176 ; 0xb0\n", - " 7b6: 23b2 movs r3, #178 ; 0xb2\n", - " 7b8: 23b4 movs r3, #180 ; 0xb4\n", - " 7ba: 23b6 movs r3, #182 ; 0xb6\n", - " 7bc: 23b8 movs r3, #184 ; 0xb8\n", - " 7be: 23ba movs r3, #186 ; 0xba\n", - " 7c0: 23bc movs r3, #188 ; 0xbc\n", - " 7c2: 23be movs r3, #190 ; 0xbe\n", - " 7c4: 23c0 movs r3, #192 ; 0xc0\n", - " 7c6: 23c2 movs r3, #194 ; 0xc2\n", - " 7c8: 23c4 movs r3, #196 ; 0xc4\n", - " 7ca: 23c6 movs r3, #198 ; 0xc6\n", - " 7cc: 23c8 movs r3, #200 ; 0xc8\n", - " 7ce: 23ca movs r3, #202 ; 0xca\n", - " 7d0: 23cc movs r3, #204 ; 0xcc\n", - " 7d2: 23ce movs r3, #206 ; 0xce\n", - " 7d4: 23d0 movs r3, #208 ; 0xd0\n", - " 7d6: 23d2 movs r3, #210 ; 0xd2\n", - " 7d8: 23d4 movs r3, #212 ; 0xd4\n", - " 7da: 23d6 movs r3, #214 ; 0xd6\n", - " 7dc: 23d8 movs r3, #216 ; 0xd8\n", - " 7de: 23da movs r3, #218 ; 0xda\n", - " 7e0: 23dc movs r3, #220 ; 0xdc\n", - " 7e2: 23de movs r3, #222 ; 0xde\n", - " 7e4: 23e0 movs r3, #224 ; 0xe0\n", - " 7e6: 23e2 movs r3, #226 ; 0xe2\n", - " 7e8: 23e4 movs r3, #228 ; 0xe4\n", - " 7ea: 23e6 movs r3, #230 ; 0xe6\n", - " 7ec: 23e8 movs r3, #232 ; 0xe8\n", - " 7ee: 23ea movs r3, #234 ; 0xea\n", - " 7f0: 23ec movs r3, #236 ; 0xec\n", - " 7f2: 23ee movs r3, #238 ; 0xee\n", - " 7f4: 23f0 movs r3, #240 ; 0xf0\n", - " 7f6: 23f2 movs r3, #242 ; 0xf2\n", - " 7f8: 23f4 movs r3, #244 ; 0xf4\n", - " 7fa: 23f6 movs r3, #246 ; 0xf6\n", - " 7fc: 23f8 movs r3, #248 ; 0xf8\n", - " 7fe: 23fa movs r3, #250 ; 0xfa\n", - " 800: 23fc movs r3, #252 ; 0xfc\n", - " 802: 23fe movs r3, #254 ; 0xfe\n", - " 804: 2300 movs r3, #0\n", - " 806: 4611 mov r1, r2\n", - nullptr -}; -const char* const CompareAndBranchMaxResults[] = { - " 0: b3fc cbz r4, 82 <CompareAndBranchMax+0x82>\n", - " 2: 2300 movs r3, #0\n", - " 4: 2302 movs r3, #2\n", - " 6: 2304 movs r3, #4\n", - " 8: 2306 movs r3, #6\n", - " a: 2308 movs r3, #8\n", - " c: 230a movs r3, #10\n", - " e: 230c movs r3, #12\n", - " 10: 230e movs r3, #14\n", - " 12: 2310 movs r3, #16\n", - " 14: 2312 movs r3, #18\n", - " 16: 2314 movs r3, #20\n", - " 18: 2316 movs r3, #22\n", - " 1a: 2318 movs r3, #24\n", - " 1c: 231a movs r3, #26\n", - " 1e: 231c movs r3, #28\n", - " 20: 231e movs r3, #30\n", - " 22: 2320 movs r3, #32\n", - " 24: 2322 movs r3, #34 ; 0x22\n", - " 26: 2324 movs r3, #36 ; 0x24\n", - " 28: 2326 movs r3, #38 ; 0x26\n", - " 2a: 2328 movs r3, #40 ; 0x28\n", - " 2c: 232a movs r3, #42 ; 0x2a\n", - " 2e: 232c movs r3, #44 ; 0x2c\n", - " 30: 232e movs r3, #46 ; 0x2e\n", - " 32: 2330 movs r3, #48 ; 0x30\n", - " 34: 2332 movs r3, #50 ; 0x32\n", - " 36: 2334 movs r3, #52 ; 0x34\n", - " 38: 2336 movs r3, #54 ; 0x36\n", - " 3a: 2338 movs r3, #56 ; 0x38\n", - " 3c: 233a movs r3, #58 ; 0x3a\n", - " 3e: 233c movs r3, #60 ; 0x3c\n", - " 40: 233e movs r3, #62 ; 0x3e\n", - " 42: 2340 movs r3, #64 ; 0x40\n", - " 44: 2342 movs r3, #66 ; 0x42\n", - " 46: 2344 movs r3, #68 ; 0x44\n", - " 48: 2346 movs r3, #70 ; 0x46\n", - " 4a: 2348 movs r3, #72 ; 0x48\n", - " 4c: 234a movs r3, #74 ; 0x4a\n", - " 4e: 234c movs r3, #76 ; 0x4c\n", - " 50: 234e movs r3, #78 ; 0x4e\n", - " 52: 2350 movs r3, #80 ; 0x50\n", - " 54: 2352 movs r3, #82 ; 0x52\n", - " 56: 2354 movs r3, #84 ; 0x54\n", - " 58: 2356 movs r3, #86 ; 0x56\n", - " 5a: 2358 movs r3, #88 ; 0x58\n", - " 5c: 235a movs r3, #90 ; 0x5a\n", - " 5e: 235c movs r3, #92 ; 0x5c\n", - " 60: 235e movs r3, #94 ; 0x5e\n", - " 62: 2360 movs r3, #96 ; 0x60\n", - " 64: 2362 movs r3, #98 ; 0x62\n", - " 66: 2364 movs r3, #100 ; 0x64\n", - " 68: 2366 movs r3, #102 ; 0x66\n", - " 6a: 2368 movs r3, #104 ; 0x68\n", - " 6c: 236a movs r3, #106 ; 0x6a\n", - " 6e: 236c movs r3, #108 ; 0x6c\n", - " 70: 236e movs r3, #110 ; 0x6e\n", - " 72: 2370 movs r3, #112 ; 0x70\n", - " 74: 2372 movs r3, #114 ; 0x72\n", - " 76: 2374 movs r3, #116 ; 0x74\n", - " 78: 2376 movs r3, #118 ; 0x76\n", - " 7a: 2378 movs r3, #120 ; 0x78\n", - " 7c: 237a movs r3, #122 ; 0x7a\n", - " 7e: 237c movs r3, #124 ; 0x7c\n", - " 80: 237e movs r3, #126 ; 0x7e\n", - " 82: 4611 mov r1, r2\n", - nullptr -}; -const char* const CompareAndBranchRelocation16Results[] = { - " 0: 2c00 cmp r4, #0\n", - " 2: d040 beq.n 86 <CompareAndBranchRelocation16+0x86>\n", - " 4: 2300 movs r3, #0\n", - " 6: 2302 movs r3, #2\n", - " 8: 2304 movs r3, #4\n", - " a: 2306 movs r3, #6\n", - " c: 2308 movs r3, #8\n", - " e: 230a movs r3, #10\n", - " 10: 230c movs r3, #12\n", - " 12: 230e movs r3, #14\n", - " 14: 2310 movs r3, #16\n", - " 16: 2312 movs r3, #18\n", - " 18: 2314 movs r3, #20\n", - " 1a: 2316 movs r3, #22\n", - " 1c: 2318 movs r3, #24\n", - " 1e: 231a movs r3, #26\n", - " 20: 231c movs r3, #28\n", - " 22: 231e movs r3, #30\n", - " 24: 2320 movs r3, #32\n", - " 26: 2322 movs r3, #34 ; 0x22\n", - " 28: 2324 movs r3, #36 ; 0x24\n", - " 2a: 2326 movs r3, #38 ; 0x26\n", - " 2c: 2328 movs r3, #40 ; 0x28\n", - " 2e: 232a movs r3, #42 ; 0x2a\n", - " 30: 232c movs r3, #44 ; 0x2c\n", - " 32: 232e movs r3, #46 ; 0x2e\n", - " 34: 2330 movs r3, #48 ; 0x30\n", - " 36: 2332 movs r3, #50 ; 0x32\n", - " 38: 2334 movs r3, #52 ; 0x34\n", - " 3a: 2336 movs r3, #54 ; 0x36\n", - " 3c: 2338 movs r3, #56 ; 0x38\n", - " 3e: 233a movs r3, #58 ; 0x3a\n", - " 40: 233c movs r3, #60 ; 0x3c\n", - " 42: 233e movs r3, #62 ; 0x3e\n", - " 44: 2340 movs r3, #64 ; 0x40\n", - " 46: 2342 movs r3, #66 ; 0x42\n", - " 48: 2344 movs r3, #68 ; 0x44\n", - " 4a: 2346 movs r3, #70 ; 0x46\n", - " 4c: 2348 movs r3, #72 ; 0x48\n", - " 4e: 234a movs r3, #74 ; 0x4a\n", - " 50: 234c movs r3, #76 ; 0x4c\n", - " 52: 234e movs r3, #78 ; 0x4e\n", - " 54: 2350 movs r3, #80 ; 0x50\n", - " 56: 2352 movs r3, #82 ; 0x52\n", - " 58: 2354 movs r3, #84 ; 0x54\n", - " 5a: 2356 movs r3, #86 ; 0x56\n", - " 5c: 2358 movs r3, #88 ; 0x58\n", - " 5e: 235a movs r3, #90 ; 0x5a\n", - " 60: 235c movs r3, #92 ; 0x5c\n", - " 62: 235e movs r3, #94 ; 0x5e\n", - " 64: 2360 movs r3, #96 ; 0x60\n", - " 66: 2362 movs r3, #98 ; 0x62\n", - " 68: 2364 movs r3, #100 ; 0x64\n", - " 6a: 2366 movs r3, #102 ; 0x66\n", - " 6c: 2368 movs r3, #104 ; 0x68\n", - " 6e: 236a movs r3, #106 ; 0x6a\n", - " 70: 236c movs r3, #108 ; 0x6c\n", - " 72: 236e movs r3, #110 ; 0x6e\n", - " 74: 2370 movs r3, #112 ; 0x70\n", - " 76: 2372 movs r3, #114 ; 0x72\n", - " 78: 2374 movs r3, #116 ; 0x74\n", - " 7a: 2376 movs r3, #118 ; 0x76\n", - " 7c: 2378 movs r3, #120 ; 0x78\n", - " 7e: 237a movs r3, #122 ; 0x7a\n", - " 80: 237c movs r3, #124 ; 0x7c\n", - " 82: 237e movs r3, #126 ; 0x7e\n", - " 84: 2380 movs r3, #128 ; 0x80\n", - " 86: 4611 mov r1, r2\n", - nullptr -}; -const char* const CompareAndBranchRelocation32Results[] = { - " 0: 2c00 cmp r4, #0\n", - " 2: f000 8401 beq.w 808 <CompareAndBranchRelocation32+0x808>\n", - " 6: 2300 movs r3, #0\n", - " 8: 2302 movs r3, #2\n", - " a: 2304 movs r3, #4\n", - " c: 2306 movs r3, #6\n", - " e: 2308 movs r3, #8\n", - " 10: 230a movs r3, #10\n", - " 12: 230c movs r3, #12\n", - " 14: 230e movs r3, #14\n", - " 16: 2310 movs r3, #16\n", - " 18: 2312 movs r3, #18\n", - " 1a: 2314 movs r3, #20\n", - " 1c: 2316 movs r3, #22\n", - " 1e: 2318 movs r3, #24\n", - " 20: 231a movs r3, #26\n", - " 22: 231c movs r3, #28\n", - " 24: 231e movs r3, #30\n", - " 26: 2320 movs r3, #32\n", - " 28: 2322 movs r3, #34 ; 0x22\n", - " 2a: 2324 movs r3, #36 ; 0x24\n", - " 2c: 2326 movs r3, #38 ; 0x26\n", - " 2e: 2328 movs r3, #40 ; 0x28\n", - " 30: 232a movs r3, #42 ; 0x2a\n", - " 32: 232c movs r3, #44 ; 0x2c\n", - " 34: 232e movs r3, #46 ; 0x2e\n", - " 36: 2330 movs r3, #48 ; 0x30\n", - " 38: 2332 movs r3, #50 ; 0x32\n", - " 3a: 2334 movs r3, #52 ; 0x34\n", - " 3c: 2336 movs r3, #54 ; 0x36\n", - " 3e: 2338 movs r3, #56 ; 0x38\n", - " 40: 233a movs r3, #58 ; 0x3a\n", - " 42: 233c movs r3, #60 ; 0x3c\n", - " 44: 233e movs r3, #62 ; 0x3e\n", - " 46: 2340 movs r3, #64 ; 0x40\n", - " 48: 2342 movs r3, #66 ; 0x42\n", - " 4a: 2344 movs r3, #68 ; 0x44\n", - " 4c: 2346 movs r3, #70 ; 0x46\n", - " 4e: 2348 movs r3, #72 ; 0x48\n", - " 50: 234a movs r3, #74 ; 0x4a\n", - " 52: 234c movs r3, #76 ; 0x4c\n", - " 54: 234e movs r3, #78 ; 0x4e\n", - " 56: 2350 movs r3, #80 ; 0x50\n", - " 58: 2352 movs r3, #82 ; 0x52\n", - " 5a: 2354 movs r3, #84 ; 0x54\n", - " 5c: 2356 movs r3, #86 ; 0x56\n", - " 5e: 2358 movs r3, #88 ; 0x58\n", - " 60: 235a movs r3, #90 ; 0x5a\n", - " 62: 235c movs r3, #92 ; 0x5c\n", - " 64: 235e movs r3, #94 ; 0x5e\n", - " 66: 2360 movs r3, #96 ; 0x60\n", - " 68: 2362 movs r3, #98 ; 0x62\n", - " 6a: 2364 movs r3, #100 ; 0x64\n", - " 6c: 2366 movs r3, #102 ; 0x66\n", - " 6e: 2368 movs r3, #104 ; 0x68\n", - " 70: 236a movs r3, #106 ; 0x6a\n", - " 72: 236c movs r3, #108 ; 0x6c\n", - " 74: 236e movs r3, #110 ; 0x6e\n", - " 76: 2370 movs r3, #112 ; 0x70\n", - " 78: 2372 movs r3, #114 ; 0x72\n", - " 7a: 2374 movs r3, #116 ; 0x74\n", - " 7c: 2376 movs r3, #118 ; 0x76\n", - " 7e: 2378 movs r3, #120 ; 0x78\n", - " 80: 237a movs r3, #122 ; 0x7a\n", - " 82: 237c movs r3, #124 ; 0x7c\n", - " 84: 237e movs r3, #126 ; 0x7e\n", - " 86: 2380 movs r3, #128 ; 0x80\n", - " 88: 2382 movs r3, #130 ; 0x82\n", - " 8a: 2384 movs r3, #132 ; 0x84\n", - " 8c: 2386 movs r3, #134 ; 0x86\n", - " 8e: 2388 movs r3, #136 ; 0x88\n", - " 90: 238a movs r3, #138 ; 0x8a\n", - " 92: 238c movs r3, #140 ; 0x8c\n", - " 94: 238e movs r3, #142 ; 0x8e\n", - " 96: 2390 movs r3, #144 ; 0x90\n", - " 98: 2392 movs r3, #146 ; 0x92\n", - " 9a: 2394 movs r3, #148 ; 0x94\n", - " 9c: 2396 movs r3, #150 ; 0x96\n", - " 9e: 2398 movs r3, #152 ; 0x98\n", - " a0: 239a movs r3, #154 ; 0x9a\n", - " a2: 239c movs r3, #156 ; 0x9c\n", - " a4: 239e movs r3, #158 ; 0x9e\n", - " a6: 23a0 movs r3, #160 ; 0xa0\n", - " a8: 23a2 movs r3, #162 ; 0xa2\n", - " aa: 23a4 movs r3, #164 ; 0xa4\n", - " ac: 23a6 movs r3, #166 ; 0xa6\n", - " ae: 23a8 movs r3, #168 ; 0xa8\n", - " b0: 23aa movs r3, #170 ; 0xaa\n", - " b2: 23ac movs r3, #172 ; 0xac\n", - " b4: 23ae movs r3, #174 ; 0xae\n", - " b6: 23b0 movs r3, #176 ; 0xb0\n", - " b8: 23b2 movs r3, #178 ; 0xb2\n", - " ba: 23b4 movs r3, #180 ; 0xb4\n", - " bc: 23b6 movs r3, #182 ; 0xb6\n", - " be: 23b8 movs r3, #184 ; 0xb8\n", - " c0: 23ba movs r3, #186 ; 0xba\n", - " c2: 23bc movs r3, #188 ; 0xbc\n", - " c4: 23be movs r3, #190 ; 0xbe\n", - " c6: 23c0 movs r3, #192 ; 0xc0\n", - " c8: 23c2 movs r3, #194 ; 0xc2\n", - " ca: 23c4 movs r3, #196 ; 0xc4\n", - " cc: 23c6 movs r3, #198 ; 0xc6\n", - " ce: 23c8 movs r3, #200 ; 0xc8\n", - " d0: 23ca movs r3, #202 ; 0xca\n", - " d2: 23cc movs r3, #204 ; 0xcc\n", - " d4: 23ce movs r3, #206 ; 0xce\n", - " d6: 23d0 movs r3, #208 ; 0xd0\n", - " d8: 23d2 movs r3, #210 ; 0xd2\n", - " da: 23d4 movs r3, #212 ; 0xd4\n", - " dc: 23d6 movs r3, #214 ; 0xd6\n", - " de: 23d8 movs r3, #216 ; 0xd8\n", - " e0: 23da movs r3, #218 ; 0xda\n", - " e2: 23dc movs r3, #220 ; 0xdc\n", - " e4: 23de movs r3, #222 ; 0xde\n", - " e6: 23e0 movs r3, #224 ; 0xe0\n", - " e8: 23e2 movs r3, #226 ; 0xe2\n", - " ea: 23e4 movs r3, #228 ; 0xe4\n", - " ec: 23e6 movs r3, #230 ; 0xe6\n", - " ee: 23e8 movs r3, #232 ; 0xe8\n", - " f0: 23ea movs r3, #234 ; 0xea\n", - " f2: 23ec movs r3, #236 ; 0xec\n", - " f4: 23ee movs r3, #238 ; 0xee\n", - " f6: 23f0 movs r3, #240 ; 0xf0\n", - " f8: 23f2 movs r3, #242 ; 0xf2\n", - " fa: 23f4 movs r3, #244 ; 0xf4\n", - " fc: 23f6 movs r3, #246 ; 0xf6\n", - " fe: 23f8 movs r3, #248 ; 0xf8\n", - " 100: 23fa movs r3, #250 ; 0xfa\n", - " 102: 23fc movs r3, #252 ; 0xfc\n", - " 104: 23fe movs r3, #254 ; 0xfe\n", - " 106: 2300 movs r3, #0\n", - " 108: 2302 movs r3, #2\n", - " 10a: 2304 movs r3, #4\n", - " 10c: 2306 movs r3, #6\n", - " 10e: 2308 movs r3, #8\n", - " 110: 230a movs r3, #10\n", - " 112: 230c movs r3, #12\n", - " 114: 230e movs r3, #14\n", - " 116: 2310 movs r3, #16\n", - " 118: 2312 movs r3, #18\n", - " 11a: 2314 movs r3, #20\n", - " 11c: 2316 movs r3, #22\n", - " 11e: 2318 movs r3, #24\n", - " 120: 231a movs r3, #26\n", - " 122: 231c movs r3, #28\n", - " 124: 231e movs r3, #30\n", - " 126: 2320 movs r3, #32\n", - " 128: 2322 movs r3, #34 ; 0x22\n", - " 12a: 2324 movs r3, #36 ; 0x24\n", - " 12c: 2326 movs r3, #38 ; 0x26\n", - " 12e: 2328 movs r3, #40 ; 0x28\n", - " 130: 232a movs r3, #42 ; 0x2a\n", - " 132: 232c movs r3, #44 ; 0x2c\n", - " 134: 232e movs r3, #46 ; 0x2e\n", - " 136: 2330 movs r3, #48 ; 0x30\n", - " 138: 2332 movs r3, #50 ; 0x32\n", - " 13a: 2334 movs r3, #52 ; 0x34\n", - " 13c: 2336 movs r3, #54 ; 0x36\n", - " 13e: 2338 movs r3, #56 ; 0x38\n", - " 140: 233a movs r3, #58 ; 0x3a\n", - " 142: 233c movs r3, #60 ; 0x3c\n", - " 144: 233e movs r3, #62 ; 0x3e\n", - " 146: 2340 movs r3, #64 ; 0x40\n", - " 148: 2342 movs r3, #66 ; 0x42\n", - " 14a: 2344 movs r3, #68 ; 0x44\n", - " 14c: 2346 movs r3, #70 ; 0x46\n", - " 14e: 2348 movs r3, #72 ; 0x48\n", - " 150: 234a movs r3, #74 ; 0x4a\n", - " 152: 234c movs r3, #76 ; 0x4c\n", - " 154: 234e movs r3, #78 ; 0x4e\n", - " 156: 2350 movs r3, #80 ; 0x50\n", - " 158: 2352 movs r3, #82 ; 0x52\n", - " 15a: 2354 movs r3, #84 ; 0x54\n", - " 15c: 2356 movs r3, #86 ; 0x56\n", - " 15e: 2358 movs r3, #88 ; 0x58\n", - " 160: 235a movs r3, #90 ; 0x5a\n", - " 162: 235c movs r3, #92 ; 0x5c\n", - " 164: 235e movs r3, #94 ; 0x5e\n", - " 166: 2360 movs r3, #96 ; 0x60\n", - " 168: 2362 movs r3, #98 ; 0x62\n", - " 16a: 2364 movs r3, #100 ; 0x64\n", - " 16c: 2366 movs r3, #102 ; 0x66\n", - " 16e: 2368 movs r3, #104 ; 0x68\n", - " 170: 236a movs r3, #106 ; 0x6a\n", - " 172: 236c movs r3, #108 ; 0x6c\n", - " 174: 236e movs r3, #110 ; 0x6e\n", - " 176: 2370 movs r3, #112 ; 0x70\n", - " 178: 2372 movs r3, #114 ; 0x72\n", - " 17a: 2374 movs r3, #116 ; 0x74\n", - " 17c: 2376 movs r3, #118 ; 0x76\n", - " 17e: 2378 movs r3, #120 ; 0x78\n", - " 180: 237a movs r3, #122 ; 0x7a\n", - " 182: 237c movs r3, #124 ; 0x7c\n", - " 184: 237e movs r3, #126 ; 0x7e\n", - " 186: 2380 movs r3, #128 ; 0x80\n", - " 188: 2382 movs r3, #130 ; 0x82\n", - " 18a: 2384 movs r3, #132 ; 0x84\n", - " 18c: 2386 movs r3, #134 ; 0x86\n", - " 18e: 2388 movs r3, #136 ; 0x88\n", - " 190: 238a movs r3, #138 ; 0x8a\n", - " 192: 238c movs r3, #140 ; 0x8c\n", - " 194: 238e movs r3, #142 ; 0x8e\n", - " 196: 2390 movs r3, #144 ; 0x90\n", - " 198: 2392 movs r3, #146 ; 0x92\n", - " 19a: 2394 movs r3, #148 ; 0x94\n", - " 19c: 2396 movs r3, #150 ; 0x96\n", - " 19e: 2398 movs r3, #152 ; 0x98\n", - " 1a0: 239a movs r3, #154 ; 0x9a\n", - " 1a2: 239c movs r3, #156 ; 0x9c\n", - " 1a4: 239e movs r3, #158 ; 0x9e\n", - " 1a6: 23a0 movs r3, #160 ; 0xa0\n", - " 1a8: 23a2 movs r3, #162 ; 0xa2\n", - " 1aa: 23a4 movs r3, #164 ; 0xa4\n", - " 1ac: 23a6 movs r3, #166 ; 0xa6\n", - " 1ae: 23a8 movs r3, #168 ; 0xa8\n", - " 1b0: 23aa movs r3, #170 ; 0xaa\n", - " 1b2: 23ac movs r3, #172 ; 0xac\n", - " 1b4: 23ae movs r3, #174 ; 0xae\n", - " 1b6: 23b0 movs r3, #176 ; 0xb0\n", - " 1b8: 23b2 movs r3, #178 ; 0xb2\n", - " 1ba: 23b4 movs r3, #180 ; 0xb4\n", - " 1bc: 23b6 movs r3, #182 ; 0xb6\n", - " 1be: 23b8 movs r3, #184 ; 0xb8\n", - " 1c0: 23ba movs r3, #186 ; 0xba\n", - " 1c2: 23bc movs r3, #188 ; 0xbc\n", - " 1c4: 23be movs r3, #190 ; 0xbe\n", - " 1c6: 23c0 movs r3, #192 ; 0xc0\n", - " 1c8: 23c2 movs r3, #194 ; 0xc2\n", - " 1ca: 23c4 movs r3, #196 ; 0xc4\n", - " 1cc: 23c6 movs r3, #198 ; 0xc6\n", - " 1ce: 23c8 movs r3, #200 ; 0xc8\n", - " 1d0: 23ca movs r3, #202 ; 0xca\n", - " 1d2: 23cc movs r3, #204 ; 0xcc\n", - " 1d4: 23ce movs r3, #206 ; 0xce\n", - " 1d6: 23d0 movs r3, #208 ; 0xd0\n", - " 1d8: 23d2 movs r3, #210 ; 0xd2\n", - " 1da: 23d4 movs r3, #212 ; 0xd4\n", - " 1dc: 23d6 movs r3, #214 ; 0xd6\n", - " 1de: 23d8 movs r3, #216 ; 0xd8\n", - " 1e0: 23da movs r3, #218 ; 0xda\n", - " 1e2: 23dc movs r3, #220 ; 0xdc\n", - " 1e4: 23de movs r3, #222 ; 0xde\n", - " 1e6: 23e0 movs r3, #224 ; 0xe0\n", - " 1e8: 23e2 movs r3, #226 ; 0xe2\n", - " 1ea: 23e4 movs r3, #228 ; 0xe4\n", - " 1ec: 23e6 movs r3, #230 ; 0xe6\n", - " 1ee: 23e8 movs r3, #232 ; 0xe8\n", - " 1f0: 23ea movs r3, #234 ; 0xea\n", - " 1f2: 23ec movs r3, #236 ; 0xec\n", - " 1f4: 23ee movs r3, #238 ; 0xee\n", - " 1f6: 23f0 movs r3, #240 ; 0xf0\n", - " 1f8: 23f2 movs r3, #242 ; 0xf2\n", - " 1fa: 23f4 movs r3, #244 ; 0xf4\n", - " 1fc: 23f6 movs r3, #246 ; 0xf6\n", - " 1fe: 23f8 movs r3, #248 ; 0xf8\n", - " 200: 23fa movs r3, #250 ; 0xfa\n", - " 202: 23fc movs r3, #252 ; 0xfc\n", - " 204: 23fe movs r3, #254 ; 0xfe\n", - " 206: 2300 movs r3, #0\n", - " 208: 2302 movs r3, #2\n", - " 20a: 2304 movs r3, #4\n", - " 20c: 2306 movs r3, #6\n", - " 20e: 2308 movs r3, #8\n", - " 210: 230a movs r3, #10\n", - " 212: 230c movs r3, #12\n", - " 214: 230e movs r3, #14\n", - " 216: 2310 movs r3, #16\n", - " 218: 2312 movs r3, #18\n", - " 21a: 2314 movs r3, #20\n", - " 21c: 2316 movs r3, #22\n", - " 21e: 2318 movs r3, #24\n", - " 220: 231a movs r3, #26\n", - " 222: 231c movs r3, #28\n", - " 224: 231e movs r3, #30\n", - " 226: 2320 movs r3, #32\n", - " 228: 2322 movs r3, #34 ; 0x22\n", - " 22a: 2324 movs r3, #36 ; 0x24\n", - " 22c: 2326 movs r3, #38 ; 0x26\n", - " 22e: 2328 movs r3, #40 ; 0x28\n", - " 230: 232a movs r3, #42 ; 0x2a\n", - " 232: 232c movs r3, #44 ; 0x2c\n", - " 234: 232e movs r3, #46 ; 0x2e\n", - " 236: 2330 movs r3, #48 ; 0x30\n", - " 238: 2332 movs r3, #50 ; 0x32\n", - " 23a: 2334 movs r3, #52 ; 0x34\n", - " 23c: 2336 movs r3, #54 ; 0x36\n", - " 23e: 2338 movs r3, #56 ; 0x38\n", - " 240: 233a movs r3, #58 ; 0x3a\n", - " 242: 233c movs r3, #60 ; 0x3c\n", - " 244: 233e movs r3, #62 ; 0x3e\n", - " 246: 2340 movs r3, #64 ; 0x40\n", - " 248: 2342 movs r3, #66 ; 0x42\n", - " 24a: 2344 movs r3, #68 ; 0x44\n", - " 24c: 2346 movs r3, #70 ; 0x46\n", - " 24e: 2348 movs r3, #72 ; 0x48\n", - " 250: 234a movs r3, #74 ; 0x4a\n", - " 252: 234c movs r3, #76 ; 0x4c\n", - " 254: 234e movs r3, #78 ; 0x4e\n", - " 256: 2350 movs r3, #80 ; 0x50\n", - " 258: 2352 movs r3, #82 ; 0x52\n", - " 25a: 2354 movs r3, #84 ; 0x54\n", - " 25c: 2356 movs r3, #86 ; 0x56\n", - " 25e: 2358 movs r3, #88 ; 0x58\n", - " 260: 235a movs r3, #90 ; 0x5a\n", - " 262: 235c movs r3, #92 ; 0x5c\n", - " 264: 235e movs r3, #94 ; 0x5e\n", - " 266: 2360 movs r3, #96 ; 0x60\n", - " 268: 2362 movs r3, #98 ; 0x62\n", - " 26a: 2364 movs r3, #100 ; 0x64\n", - " 26c: 2366 movs r3, #102 ; 0x66\n", - " 26e: 2368 movs r3, #104 ; 0x68\n", - " 270: 236a movs r3, #106 ; 0x6a\n", - " 272: 236c movs r3, #108 ; 0x6c\n", - " 274: 236e movs r3, #110 ; 0x6e\n", - " 276: 2370 movs r3, #112 ; 0x70\n", - " 278: 2372 movs r3, #114 ; 0x72\n", - " 27a: 2374 movs r3, #116 ; 0x74\n", - " 27c: 2376 movs r3, #118 ; 0x76\n", - " 27e: 2378 movs r3, #120 ; 0x78\n", - " 280: 237a movs r3, #122 ; 0x7a\n", - " 282: 237c movs r3, #124 ; 0x7c\n", - " 284: 237e movs r3, #126 ; 0x7e\n", - " 286: 2380 movs r3, #128 ; 0x80\n", - " 288: 2382 movs r3, #130 ; 0x82\n", - " 28a: 2384 movs r3, #132 ; 0x84\n", - " 28c: 2386 movs r3, #134 ; 0x86\n", - " 28e: 2388 movs r3, #136 ; 0x88\n", - " 290: 238a movs r3, #138 ; 0x8a\n", - " 292: 238c movs r3, #140 ; 0x8c\n", - " 294: 238e movs r3, #142 ; 0x8e\n", - " 296: 2390 movs r3, #144 ; 0x90\n", - " 298: 2392 movs r3, #146 ; 0x92\n", - " 29a: 2394 movs r3, #148 ; 0x94\n", - " 29c: 2396 movs r3, #150 ; 0x96\n", - " 29e: 2398 movs r3, #152 ; 0x98\n", - " 2a0: 239a movs r3, #154 ; 0x9a\n", - " 2a2: 239c movs r3, #156 ; 0x9c\n", - " 2a4: 239e movs r3, #158 ; 0x9e\n", - " 2a6: 23a0 movs r3, #160 ; 0xa0\n", - " 2a8: 23a2 movs r3, #162 ; 0xa2\n", - " 2aa: 23a4 movs r3, #164 ; 0xa4\n", - " 2ac: 23a6 movs r3, #166 ; 0xa6\n", - " 2ae: 23a8 movs r3, #168 ; 0xa8\n", - " 2b0: 23aa movs r3, #170 ; 0xaa\n", - " 2b2: 23ac movs r3, #172 ; 0xac\n", - " 2b4: 23ae movs r3, #174 ; 0xae\n", - " 2b6: 23b0 movs r3, #176 ; 0xb0\n", - " 2b8: 23b2 movs r3, #178 ; 0xb2\n", - " 2ba: 23b4 movs r3, #180 ; 0xb4\n", - " 2bc: 23b6 movs r3, #182 ; 0xb6\n", - " 2be: 23b8 movs r3, #184 ; 0xb8\n", - " 2c0: 23ba movs r3, #186 ; 0xba\n", - " 2c2: 23bc movs r3, #188 ; 0xbc\n", - " 2c4: 23be movs r3, #190 ; 0xbe\n", - " 2c6: 23c0 movs r3, #192 ; 0xc0\n", - " 2c8: 23c2 movs r3, #194 ; 0xc2\n", - " 2ca: 23c4 movs r3, #196 ; 0xc4\n", - " 2cc: 23c6 movs r3, #198 ; 0xc6\n", - " 2ce: 23c8 movs r3, #200 ; 0xc8\n", - " 2d0: 23ca movs r3, #202 ; 0xca\n", - " 2d2: 23cc movs r3, #204 ; 0xcc\n", - " 2d4: 23ce movs r3, #206 ; 0xce\n", - " 2d6: 23d0 movs r3, #208 ; 0xd0\n", - " 2d8: 23d2 movs r3, #210 ; 0xd2\n", - " 2da: 23d4 movs r3, #212 ; 0xd4\n", - " 2dc: 23d6 movs r3, #214 ; 0xd6\n", - " 2de: 23d8 movs r3, #216 ; 0xd8\n", - " 2e0: 23da movs r3, #218 ; 0xda\n", - " 2e2: 23dc movs r3, #220 ; 0xdc\n", - " 2e4: 23de movs r3, #222 ; 0xde\n", - " 2e6: 23e0 movs r3, #224 ; 0xe0\n", - " 2e8: 23e2 movs r3, #226 ; 0xe2\n", - " 2ea: 23e4 movs r3, #228 ; 0xe4\n", - " 2ec: 23e6 movs r3, #230 ; 0xe6\n", - " 2ee: 23e8 movs r3, #232 ; 0xe8\n", - " 2f0: 23ea movs r3, #234 ; 0xea\n", - " 2f2: 23ec movs r3, #236 ; 0xec\n", - " 2f4: 23ee movs r3, #238 ; 0xee\n", - " 2f6: 23f0 movs r3, #240 ; 0xf0\n", - " 2f8: 23f2 movs r3, #242 ; 0xf2\n", - " 2fa: 23f4 movs r3, #244 ; 0xf4\n", - " 2fc: 23f6 movs r3, #246 ; 0xf6\n", - " 2fe: 23f8 movs r3, #248 ; 0xf8\n", - " 300: 23fa movs r3, #250 ; 0xfa\n", - " 302: 23fc movs r3, #252 ; 0xfc\n", - " 304: 23fe movs r3, #254 ; 0xfe\n", - " 306: 2300 movs r3, #0\n", - " 308: 2302 movs r3, #2\n", - " 30a: 2304 movs r3, #4\n", - " 30c: 2306 movs r3, #6\n", - " 30e: 2308 movs r3, #8\n", - " 310: 230a movs r3, #10\n", - " 312: 230c movs r3, #12\n", - " 314: 230e movs r3, #14\n", - " 316: 2310 movs r3, #16\n", - " 318: 2312 movs r3, #18\n", - " 31a: 2314 movs r3, #20\n", - " 31c: 2316 movs r3, #22\n", - " 31e: 2318 movs r3, #24\n", - " 320: 231a movs r3, #26\n", - " 322: 231c movs r3, #28\n", - " 324: 231e movs r3, #30\n", - " 326: 2320 movs r3, #32\n", - " 328: 2322 movs r3, #34 ; 0x22\n", - " 32a: 2324 movs r3, #36 ; 0x24\n", - " 32c: 2326 movs r3, #38 ; 0x26\n", - " 32e: 2328 movs r3, #40 ; 0x28\n", - " 330: 232a movs r3, #42 ; 0x2a\n", - " 332: 232c movs r3, #44 ; 0x2c\n", - " 334: 232e movs r3, #46 ; 0x2e\n", - " 336: 2330 movs r3, #48 ; 0x30\n", - " 338: 2332 movs r3, #50 ; 0x32\n", - " 33a: 2334 movs r3, #52 ; 0x34\n", - " 33c: 2336 movs r3, #54 ; 0x36\n", - " 33e: 2338 movs r3, #56 ; 0x38\n", - " 340: 233a movs r3, #58 ; 0x3a\n", - " 342: 233c movs r3, #60 ; 0x3c\n", - " 344: 233e movs r3, #62 ; 0x3e\n", - " 346: 2340 movs r3, #64 ; 0x40\n", - " 348: 2342 movs r3, #66 ; 0x42\n", - " 34a: 2344 movs r3, #68 ; 0x44\n", - " 34c: 2346 movs r3, #70 ; 0x46\n", - " 34e: 2348 movs r3, #72 ; 0x48\n", - " 350: 234a movs r3, #74 ; 0x4a\n", - " 352: 234c movs r3, #76 ; 0x4c\n", - " 354: 234e movs r3, #78 ; 0x4e\n", - " 356: 2350 movs r3, #80 ; 0x50\n", - " 358: 2352 movs r3, #82 ; 0x52\n", - " 35a: 2354 movs r3, #84 ; 0x54\n", - " 35c: 2356 movs r3, #86 ; 0x56\n", - " 35e: 2358 movs r3, #88 ; 0x58\n", - " 360: 235a movs r3, #90 ; 0x5a\n", - " 362: 235c movs r3, #92 ; 0x5c\n", - " 364: 235e movs r3, #94 ; 0x5e\n", - " 366: 2360 movs r3, #96 ; 0x60\n", - " 368: 2362 movs r3, #98 ; 0x62\n", - " 36a: 2364 movs r3, #100 ; 0x64\n", - " 36c: 2366 movs r3, #102 ; 0x66\n", - " 36e: 2368 movs r3, #104 ; 0x68\n", - " 370: 236a movs r3, #106 ; 0x6a\n", - " 372: 236c movs r3, #108 ; 0x6c\n", - " 374: 236e movs r3, #110 ; 0x6e\n", - " 376: 2370 movs r3, #112 ; 0x70\n", - " 378: 2372 movs r3, #114 ; 0x72\n", - " 37a: 2374 movs r3, #116 ; 0x74\n", - " 37c: 2376 movs r3, #118 ; 0x76\n", - " 37e: 2378 movs r3, #120 ; 0x78\n", - " 380: 237a movs r3, #122 ; 0x7a\n", - " 382: 237c movs r3, #124 ; 0x7c\n", - " 384: 237e movs r3, #126 ; 0x7e\n", - " 386: 2380 movs r3, #128 ; 0x80\n", - " 388: 2382 movs r3, #130 ; 0x82\n", - " 38a: 2384 movs r3, #132 ; 0x84\n", - " 38c: 2386 movs r3, #134 ; 0x86\n", - " 38e: 2388 movs r3, #136 ; 0x88\n", - " 390: 238a movs r3, #138 ; 0x8a\n", - " 392: 238c movs r3, #140 ; 0x8c\n", - " 394: 238e movs r3, #142 ; 0x8e\n", - " 396: 2390 movs r3, #144 ; 0x90\n", - " 398: 2392 movs r3, #146 ; 0x92\n", - " 39a: 2394 movs r3, #148 ; 0x94\n", - " 39c: 2396 movs r3, #150 ; 0x96\n", - " 39e: 2398 movs r3, #152 ; 0x98\n", - " 3a0: 239a movs r3, #154 ; 0x9a\n", - " 3a2: 239c movs r3, #156 ; 0x9c\n", - " 3a4: 239e movs r3, #158 ; 0x9e\n", - " 3a6: 23a0 movs r3, #160 ; 0xa0\n", - " 3a8: 23a2 movs r3, #162 ; 0xa2\n", - " 3aa: 23a4 movs r3, #164 ; 0xa4\n", - " 3ac: 23a6 movs r3, #166 ; 0xa6\n", - " 3ae: 23a8 movs r3, #168 ; 0xa8\n", - " 3b0: 23aa movs r3, #170 ; 0xaa\n", - " 3b2: 23ac movs r3, #172 ; 0xac\n", - " 3b4: 23ae movs r3, #174 ; 0xae\n", - " 3b6: 23b0 movs r3, #176 ; 0xb0\n", - " 3b8: 23b2 movs r3, #178 ; 0xb2\n", - " 3ba: 23b4 movs r3, #180 ; 0xb4\n", - " 3bc: 23b6 movs r3, #182 ; 0xb6\n", - " 3be: 23b8 movs r3, #184 ; 0xb8\n", - " 3c0: 23ba movs r3, #186 ; 0xba\n", - " 3c2: 23bc movs r3, #188 ; 0xbc\n", - " 3c4: 23be movs r3, #190 ; 0xbe\n", - " 3c6: 23c0 movs r3, #192 ; 0xc0\n", - " 3c8: 23c2 movs r3, #194 ; 0xc2\n", - " 3ca: 23c4 movs r3, #196 ; 0xc4\n", - " 3cc: 23c6 movs r3, #198 ; 0xc6\n", - " 3ce: 23c8 movs r3, #200 ; 0xc8\n", - " 3d0: 23ca movs r3, #202 ; 0xca\n", - " 3d2: 23cc movs r3, #204 ; 0xcc\n", - " 3d4: 23ce movs r3, #206 ; 0xce\n", - " 3d6: 23d0 movs r3, #208 ; 0xd0\n", - " 3d8: 23d2 movs r3, #210 ; 0xd2\n", - " 3da: 23d4 movs r3, #212 ; 0xd4\n", - " 3dc: 23d6 movs r3, #214 ; 0xd6\n", - " 3de: 23d8 movs r3, #216 ; 0xd8\n", - " 3e0: 23da movs r3, #218 ; 0xda\n", - " 3e2: 23dc movs r3, #220 ; 0xdc\n", - " 3e4: 23de movs r3, #222 ; 0xde\n", - " 3e6: 23e0 movs r3, #224 ; 0xe0\n", - " 3e8: 23e2 movs r3, #226 ; 0xe2\n", - " 3ea: 23e4 movs r3, #228 ; 0xe4\n", - " 3ec: 23e6 movs r3, #230 ; 0xe6\n", - " 3ee: 23e8 movs r3, #232 ; 0xe8\n", - " 3f0: 23ea movs r3, #234 ; 0xea\n", - " 3f2: 23ec movs r3, #236 ; 0xec\n", - " 3f4: 23ee movs r3, #238 ; 0xee\n", - " 3f6: 23f0 movs r3, #240 ; 0xf0\n", - " 3f8: 23f2 movs r3, #242 ; 0xf2\n", - " 3fa: 23f4 movs r3, #244 ; 0xf4\n", - " 3fc: 23f6 movs r3, #246 ; 0xf6\n", - " 3fe: 23f8 movs r3, #248 ; 0xf8\n", - " 400: 23fa movs r3, #250 ; 0xfa\n", - " 402: 23fc movs r3, #252 ; 0xfc\n", - " 404: 23fe movs r3, #254 ; 0xfe\n", - " 406: 2300 movs r3, #0\n", - " 408: 2302 movs r3, #2\n", - " 40a: 2304 movs r3, #4\n", - " 40c: 2306 movs r3, #6\n", - " 40e: 2308 movs r3, #8\n", - " 410: 230a movs r3, #10\n", - " 412: 230c movs r3, #12\n", - " 414: 230e movs r3, #14\n", - " 416: 2310 movs r3, #16\n", - " 418: 2312 movs r3, #18\n", - " 41a: 2314 movs r3, #20\n", - " 41c: 2316 movs r3, #22\n", - " 41e: 2318 movs r3, #24\n", - " 420: 231a movs r3, #26\n", - " 422: 231c movs r3, #28\n", - " 424: 231e movs r3, #30\n", - " 426: 2320 movs r3, #32\n", - " 428: 2322 movs r3, #34 ; 0x22\n", - " 42a: 2324 movs r3, #36 ; 0x24\n", - " 42c: 2326 movs r3, #38 ; 0x26\n", - " 42e: 2328 movs r3, #40 ; 0x28\n", - " 430: 232a movs r3, #42 ; 0x2a\n", - " 432: 232c movs r3, #44 ; 0x2c\n", - " 434: 232e movs r3, #46 ; 0x2e\n", - " 436: 2330 movs r3, #48 ; 0x30\n", - " 438: 2332 movs r3, #50 ; 0x32\n", - " 43a: 2334 movs r3, #52 ; 0x34\n", - " 43c: 2336 movs r3, #54 ; 0x36\n", - " 43e: 2338 movs r3, #56 ; 0x38\n", - " 440: 233a movs r3, #58 ; 0x3a\n", - " 442: 233c movs r3, #60 ; 0x3c\n", - " 444: 233e movs r3, #62 ; 0x3e\n", - " 446: 2340 movs r3, #64 ; 0x40\n", - " 448: 2342 movs r3, #66 ; 0x42\n", - " 44a: 2344 movs r3, #68 ; 0x44\n", - " 44c: 2346 movs r3, #70 ; 0x46\n", - " 44e: 2348 movs r3, #72 ; 0x48\n", - " 450: 234a movs r3, #74 ; 0x4a\n", - " 452: 234c movs r3, #76 ; 0x4c\n", - " 454: 234e movs r3, #78 ; 0x4e\n", - " 456: 2350 movs r3, #80 ; 0x50\n", - " 458: 2352 movs r3, #82 ; 0x52\n", - " 45a: 2354 movs r3, #84 ; 0x54\n", - " 45c: 2356 movs r3, #86 ; 0x56\n", - " 45e: 2358 movs r3, #88 ; 0x58\n", - " 460: 235a movs r3, #90 ; 0x5a\n", - " 462: 235c movs r3, #92 ; 0x5c\n", - " 464: 235e movs r3, #94 ; 0x5e\n", - " 466: 2360 movs r3, #96 ; 0x60\n", - " 468: 2362 movs r3, #98 ; 0x62\n", - " 46a: 2364 movs r3, #100 ; 0x64\n", - " 46c: 2366 movs r3, #102 ; 0x66\n", - " 46e: 2368 movs r3, #104 ; 0x68\n", - " 470: 236a movs r3, #106 ; 0x6a\n", - " 472: 236c movs r3, #108 ; 0x6c\n", - " 474: 236e movs r3, #110 ; 0x6e\n", - " 476: 2370 movs r3, #112 ; 0x70\n", - " 478: 2372 movs r3, #114 ; 0x72\n", - " 47a: 2374 movs r3, #116 ; 0x74\n", - " 47c: 2376 movs r3, #118 ; 0x76\n", - " 47e: 2378 movs r3, #120 ; 0x78\n", - " 480: 237a movs r3, #122 ; 0x7a\n", - " 482: 237c movs r3, #124 ; 0x7c\n", - " 484: 237e movs r3, #126 ; 0x7e\n", - " 486: 2380 movs r3, #128 ; 0x80\n", - " 488: 2382 movs r3, #130 ; 0x82\n", - " 48a: 2384 movs r3, #132 ; 0x84\n", - " 48c: 2386 movs r3, #134 ; 0x86\n", - " 48e: 2388 movs r3, #136 ; 0x88\n", - " 490: 238a movs r3, #138 ; 0x8a\n", - " 492: 238c movs r3, #140 ; 0x8c\n", - " 494: 238e movs r3, #142 ; 0x8e\n", - " 496: 2390 movs r3, #144 ; 0x90\n", - " 498: 2392 movs r3, #146 ; 0x92\n", - " 49a: 2394 movs r3, #148 ; 0x94\n", - " 49c: 2396 movs r3, #150 ; 0x96\n", - " 49e: 2398 movs r3, #152 ; 0x98\n", - " 4a0: 239a movs r3, #154 ; 0x9a\n", - " 4a2: 239c movs r3, #156 ; 0x9c\n", - " 4a4: 239e movs r3, #158 ; 0x9e\n", - " 4a6: 23a0 movs r3, #160 ; 0xa0\n", - " 4a8: 23a2 movs r3, #162 ; 0xa2\n", - " 4aa: 23a4 movs r3, #164 ; 0xa4\n", - " 4ac: 23a6 movs r3, #166 ; 0xa6\n", - " 4ae: 23a8 movs r3, #168 ; 0xa8\n", - " 4b0: 23aa movs r3, #170 ; 0xaa\n", - " 4b2: 23ac movs r3, #172 ; 0xac\n", - " 4b4: 23ae movs r3, #174 ; 0xae\n", - " 4b6: 23b0 movs r3, #176 ; 0xb0\n", - " 4b8: 23b2 movs r3, #178 ; 0xb2\n", - " 4ba: 23b4 movs r3, #180 ; 0xb4\n", - " 4bc: 23b6 movs r3, #182 ; 0xb6\n", - " 4be: 23b8 movs r3, #184 ; 0xb8\n", - " 4c0: 23ba movs r3, #186 ; 0xba\n", - " 4c2: 23bc movs r3, #188 ; 0xbc\n", - " 4c4: 23be movs r3, #190 ; 0xbe\n", - " 4c6: 23c0 movs r3, #192 ; 0xc0\n", - " 4c8: 23c2 movs r3, #194 ; 0xc2\n", - " 4ca: 23c4 movs r3, #196 ; 0xc4\n", - " 4cc: 23c6 movs r3, #198 ; 0xc6\n", - " 4ce: 23c8 movs r3, #200 ; 0xc8\n", - " 4d0: 23ca movs r3, #202 ; 0xca\n", - " 4d2: 23cc movs r3, #204 ; 0xcc\n", - " 4d4: 23ce movs r3, #206 ; 0xce\n", - " 4d6: 23d0 movs r3, #208 ; 0xd0\n", - " 4d8: 23d2 movs r3, #210 ; 0xd2\n", - " 4da: 23d4 movs r3, #212 ; 0xd4\n", - " 4dc: 23d6 movs r3, #214 ; 0xd6\n", - " 4de: 23d8 movs r3, #216 ; 0xd8\n", - " 4e0: 23da movs r3, #218 ; 0xda\n", - " 4e2: 23dc movs r3, #220 ; 0xdc\n", - " 4e4: 23de movs r3, #222 ; 0xde\n", - " 4e6: 23e0 movs r3, #224 ; 0xe0\n", - " 4e8: 23e2 movs r3, #226 ; 0xe2\n", - " 4ea: 23e4 movs r3, #228 ; 0xe4\n", - " 4ec: 23e6 movs r3, #230 ; 0xe6\n", - " 4ee: 23e8 movs r3, #232 ; 0xe8\n", - " 4f0: 23ea movs r3, #234 ; 0xea\n", - " 4f2: 23ec movs r3, #236 ; 0xec\n", - " 4f4: 23ee movs r3, #238 ; 0xee\n", - " 4f6: 23f0 movs r3, #240 ; 0xf0\n", - " 4f8: 23f2 movs r3, #242 ; 0xf2\n", - " 4fa: 23f4 movs r3, #244 ; 0xf4\n", - " 4fc: 23f6 movs r3, #246 ; 0xf6\n", - " 4fe: 23f8 movs r3, #248 ; 0xf8\n", - " 500: 23fa movs r3, #250 ; 0xfa\n", - " 502: 23fc movs r3, #252 ; 0xfc\n", - " 504: 23fe movs r3, #254 ; 0xfe\n", - " 506: 2300 movs r3, #0\n", - " 508: 2302 movs r3, #2\n", - " 50a: 2304 movs r3, #4\n", - " 50c: 2306 movs r3, #6\n", - " 50e: 2308 movs r3, #8\n", - " 510: 230a movs r3, #10\n", - " 512: 230c movs r3, #12\n", - " 514: 230e movs r3, #14\n", - " 516: 2310 movs r3, #16\n", - " 518: 2312 movs r3, #18\n", - " 51a: 2314 movs r3, #20\n", - " 51c: 2316 movs r3, #22\n", - " 51e: 2318 movs r3, #24\n", - " 520: 231a movs r3, #26\n", - " 522: 231c movs r3, #28\n", - " 524: 231e movs r3, #30\n", - " 526: 2320 movs r3, #32\n", - " 528: 2322 movs r3, #34 ; 0x22\n", - " 52a: 2324 movs r3, #36 ; 0x24\n", - " 52c: 2326 movs r3, #38 ; 0x26\n", - " 52e: 2328 movs r3, #40 ; 0x28\n", - " 530: 232a movs r3, #42 ; 0x2a\n", - " 532: 232c movs r3, #44 ; 0x2c\n", - " 534: 232e movs r3, #46 ; 0x2e\n", - " 536: 2330 movs r3, #48 ; 0x30\n", - " 538: 2332 movs r3, #50 ; 0x32\n", - " 53a: 2334 movs r3, #52 ; 0x34\n", - " 53c: 2336 movs r3, #54 ; 0x36\n", - " 53e: 2338 movs r3, #56 ; 0x38\n", - " 540: 233a movs r3, #58 ; 0x3a\n", - " 542: 233c movs r3, #60 ; 0x3c\n", - " 544: 233e movs r3, #62 ; 0x3e\n", - " 546: 2340 movs r3, #64 ; 0x40\n", - " 548: 2342 movs r3, #66 ; 0x42\n", - " 54a: 2344 movs r3, #68 ; 0x44\n", - " 54c: 2346 movs r3, #70 ; 0x46\n", - " 54e: 2348 movs r3, #72 ; 0x48\n", - " 550: 234a movs r3, #74 ; 0x4a\n", - " 552: 234c movs r3, #76 ; 0x4c\n", - " 554: 234e movs r3, #78 ; 0x4e\n", - " 556: 2350 movs r3, #80 ; 0x50\n", - " 558: 2352 movs r3, #82 ; 0x52\n", - " 55a: 2354 movs r3, #84 ; 0x54\n", - " 55c: 2356 movs r3, #86 ; 0x56\n", - " 55e: 2358 movs r3, #88 ; 0x58\n", - " 560: 235a movs r3, #90 ; 0x5a\n", - " 562: 235c movs r3, #92 ; 0x5c\n", - " 564: 235e movs r3, #94 ; 0x5e\n", - " 566: 2360 movs r3, #96 ; 0x60\n", - " 568: 2362 movs r3, #98 ; 0x62\n", - " 56a: 2364 movs r3, #100 ; 0x64\n", - " 56c: 2366 movs r3, #102 ; 0x66\n", - " 56e: 2368 movs r3, #104 ; 0x68\n", - " 570: 236a movs r3, #106 ; 0x6a\n", - " 572: 236c movs r3, #108 ; 0x6c\n", - " 574: 236e movs r3, #110 ; 0x6e\n", - " 576: 2370 movs r3, #112 ; 0x70\n", - " 578: 2372 movs r3, #114 ; 0x72\n", - " 57a: 2374 movs r3, #116 ; 0x74\n", - " 57c: 2376 movs r3, #118 ; 0x76\n", - " 57e: 2378 movs r3, #120 ; 0x78\n", - " 580: 237a movs r3, #122 ; 0x7a\n", - " 582: 237c movs r3, #124 ; 0x7c\n", - " 584: 237e movs r3, #126 ; 0x7e\n", - " 586: 2380 movs r3, #128 ; 0x80\n", - " 588: 2382 movs r3, #130 ; 0x82\n", - " 58a: 2384 movs r3, #132 ; 0x84\n", - " 58c: 2386 movs r3, #134 ; 0x86\n", - " 58e: 2388 movs r3, #136 ; 0x88\n", - " 590: 238a movs r3, #138 ; 0x8a\n", - " 592: 238c movs r3, #140 ; 0x8c\n", - " 594: 238e movs r3, #142 ; 0x8e\n", - " 596: 2390 movs r3, #144 ; 0x90\n", - " 598: 2392 movs r3, #146 ; 0x92\n", - " 59a: 2394 movs r3, #148 ; 0x94\n", - " 59c: 2396 movs r3, #150 ; 0x96\n", - " 59e: 2398 movs r3, #152 ; 0x98\n", - " 5a0: 239a movs r3, #154 ; 0x9a\n", - " 5a2: 239c movs r3, #156 ; 0x9c\n", - " 5a4: 239e movs r3, #158 ; 0x9e\n", - " 5a6: 23a0 movs r3, #160 ; 0xa0\n", - " 5a8: 23a2 movs r3, #162 ; 0xa2\n", - " 5aa: 23a4 movs r3, #164 ; 0xa4\n", - " 5ac: 23a6 movs r3, #166 ; 0xa6\n", - " 5ae: 23a8 movs r3, #168 ; 0xa8\n", - " 5b0: 23aa movs r3, #170 ; 0xaa\n", - " 5b2: 23ac movs r3, #172 ; 0xac\n", - " 5b4: 23ae movs r3, #174 ; 0xae\n", - " 5b6: 23b0 movs r3, #176 ; 0xb0\n", - " 5b8: 23b2 movs r3, #178 ; 0xb2\n", - " 5ba: 23b4 movs r3, #180 ; 0xb4\n", - " 5bc: 23b6 movs r3, #182 ; 0xb6\n", - " 5be: 23b8 movs r3, #184 ; 0xb8\n", - " 5c0: 23ba movs r3, #186 ; 0xba\n", - " 5c2: 23bc movs r3, #188 ; 0xbc\n", - " 5c4: 23be movs r3, #190 ; 0xbe\n", - " 5c6: 23c0 movs r3, #192 ; 0xc0\n", - " 5c8: 23c2 movs r3, #194 ; 0xc2\n", - " 5ca: 23c4 movs r3, #196 ; 0xc4\n", - " 5cc: 23c6 movs r3, #198 ; 0xc6\n", - " 5ce: 23c8 movs r3, #200 ; 0xc8\n", - " 5d0: 23ca movs r3, #202 ; 0xca\n", - " 5d2: 23cc movs r3, #204 ; 0xcc\n", - " 5d4: 23ce movs r3, #206 ; 0xce\n", - " 5d6: 23d0 movs r3, #208 ; 0xd0\n", - " 5d8: 23d2 movs r3, #210 ; 0xd2\n", - " 5da: 23d4 movs r3, #212 ; 0xd4\n", - " 5dc: 23d6 movs r3, #214 ; 0xd6\n", - " 5de: 23d8 movs r3, #216 ; 0xd8\n", - " 5e0: 23da movs r3, #218 ; 0xda\n", - " 5e2: 23dc movs r3, #220 ; 0xdc\n", - " 5e4: 23de movs r3, #222 ; 0xde\n", - " 5e6: 23e0 movs r3, #224 ; 0xe0\n", - " 5e8: 23e2 movs r3, #226 ; 0xe2\n", - " 5ea: 23e4 movs r3, #228 ; 0xe4\n", - " 5ec: 23e6 movs r3, #230 ; 0xe6\n", - " 5ee: 23e8 movs r3, #232 ; 0xe8\n", - " 5f0: 23ea movs r3, #234 ; 0xea\n", - " 5f2: 23ec movs r3, #236 ; 0xec\n", - " 5f4: 23ee movs r3, #238 ; 0xee\n", - " 5f6: 23f0 movs r3, #240 ; 0xf0\n", - " 5f8: 23f2 movs r3, #242 ; 0xf2\n", - " 5fa: 23f4 movs r3, #244 ; 0xf4\n", - " 5fc: 23f6 movs r3, #246 ; 0xf6\n", - " 5fe: 23f8 movs r3, #248 ; 0xf8\n", - " 600: 23fa movs r3, #250 ; 0xfa\n", - " 602: 23fc movs r3, #252 ; 0xfc\n", - " 604: 23fe movs r3, #254 ; 0xfe\n", - " 606: 2300 movs r3, #0\n", - " 608: 2302 movs r3, #2\n", - " 60a: 2304 movs r3, #4\n", - " 60c: 2306 movs r3, #6\n", - " 60e: 2308 movs r3, #8\n", - " 610: 230a movs r3, #10\n", - " 612: 230c movs r3, #12\n", - " 614: 230e movs r3, #14\n", - " 616: 2310 movs r3, #16\n", - " 618: 2312 movs r3, #18\n", - " 61a: 2314 movs r3, #20\n", - " 61c: 2316 movs r3, #22\n", - " 61e: 2318 movs r3, #24\n", - " 620: 231a movs r3, #26\n", - " 622: 231c movs r3, #28\n", - " 624: 231e movs r3, #30\n", - " 626: 2320 movs r3, #32\n", - " 628: 2322 movs r3, #34 ; 0x22\n", - " 62a: 2324 movs r3, #36 ; 0x24\n", - " 62c: 2326 movs r3, #38 ; 0x26\n", - " 62e: 2328 movs r3, #40 ; 0x28\n", - " 630: 232a movs r3, #42 ; 0x2a\n", - " 632: 232c movs r3, #44 ; 0x2c\n", - " 634: 232e movs r3, #46 ; 0x2e\n", - " 636: 2330 movs r3, #48 ; 0x30\n", - " 638: 2332 movs r3, #50 ; 0x32\n", - " 63a: 2334 movs r3, #52 ; 0x34\n", - " 63c: 2336 movs r3, #54 ; 0x36\n", - " 63e: 2338 movs r3, #56 ; 0x38\n", - " 640: 233a movs r3, #58 ; 0x3a\n", - " 642: 233c movs r3, #60 ; 0x3c\n", - " 644: 233e movs r3, #62 ; 0x3e\n", - " 646: 2340 movs r3, #64 ; 0x40\n", - " 648: 2342 movs r3, #66 ; 0x42\n", - " 64a: 2344 movs r3, #68 ; 0x44\n", - " 64c: 2346 movs r3, #70 ; 0x46\n", - " 64e: 2348 movs r3, #72 ; 0x48\n", - " 650: 234a movs r3, #74 ; 0x4a\n", - " 652: 234c movs r3, #76 ; 0x4c\n", - " 654: 234e movs r3, #78 ; 0x4e\n", - " 656: 2350 movs r3, #80 ; 0x50\n", - " 658: 2352 movs r3, #82 ; 0x52\n", - " 65a: 2354 movs r3, #84 ; 0x54\n", - " 65c: 2356 movs r3, #86 ; 0x56\n", - " 65e: 2358 movs r3, #88 ; 0x58\n", - " 660: 235a movs r3, #90 ; 0x5a\n", - " 662: 235c movs r3, #92 ; 0x5c\n", - " 664: 235e movs r3, #94 ; 0x5e\n", - " 666: 2360 movs r3, #96 ; 0x60\n", - " 668: 2362 movs r3, #98 ; 0x62\n", - " 66a: 2364 movs r3, #100 ; 0x64\n", - " 66c: 2366 movs r3, #102 ; 0x66\n", - " 66e: 2368 movs r3, #104 ; 0x68\n", - " 670: 236a movs r3, #106 ; 0x6a\n", - " 672: 236c movs r3, #108 ; 0x6c\n", - " 674: 236e movs r3, #110 ; 0x6e\n", - " 676: 2370 movs r3, #112 ; 0x70\n", - " 678: 2372 movs r3, #114 ; 0x72\n", - " 67a: 2374 movs r3, #116 ; 0x74\n", - " 67c: 2376 movs r3, #118 ; 0x76\n", - " 67e: 2378 movs r3, #120 ; 0x78\n", - " 680: 237a movs r3, #122 ; 0x7a\n", - " 682: 237c movs r3, #124 ; 0x7c\n", - " 684: 237e movs r3, #126 ; 0x7e\n", - " 686: 2380 movs r3, #128 ; 0x80\n", - " 688: 2382 movs r3, #130 ; 0x82\n", - " 68a: 2384 movs r3, #132 ; 0x84\n", - " 68c: 2386 movs r3, #134 ; 0x86\n", - " 68e: 2388 movs r3, #136 ; 0x88\n", - " 690: 238a movs r3, #138 ; 0x8a\n", - " 692: 238c movs r3, #140 ; 0x8c\n", - " 694: 238e movs r3, #142 ; 0x8e\n", - " 696: 2390 movs r3, #144 ; 0x90\n", - " 698: 2392 movs r3, #146 ; 0x92\n", - " 69a: 2394 movs r3, #148 ; 0x94\n", - " 69c: 2396 movs r3, #150 ; 0x96\n", - " 69e: 2398 movs r3, #152 ; 0x98\n", - " 6a0: 239a movs r3, #154 ; 0x9a\n", - " 6a2: 239c movs r3, #156 ; 0x9c\n", - " 6a4: 239e movs r3, #158 ; 0x9e\n", - " 6a6: 23a0 movs r3, #160 ; 0xa0\n", - " 6a8: 23a2 movs r3, #162 ; 0xa2\n", - " 6aa: 23a4 movs r3, #164 ; 0xa4\n", - " 6ac: 23a6 movs r3, #166 ; 0xa6\n", - " 6ae: 23a8 movs r3, #168 ; 0xa8\n", - " 6b0: 23aa movs r3, #170 ; 0xaa\n", - " 6b2: 23ac movs r3, #172 ; 0xac\n", - " 6b4: 23ae movs r3, #174 ; 0xae\n", - " 6b6: 23b0 movs r3, #176 ; 0xb0\n", - " 6b8: 23b2 movs r3, #178 ; 0xb2\n", - " 6ba: 23b4 movs r3, #180 ; 0xb4\n", - " 6bc: 23b6 movs r3, #182 ; 0xb6\n", - " 6be: 23b8 movs r3, #184 ; 0xb8\n", - " 6c0: 23ba movs r3, #186 ; 0xba\n", - " 6c2: 23bc movs r3, #188 ; 0xbc\n", - " 6c4: 23be movs r3, #190 ; 0xbe\n", - " 6c6: 23c0 movs r3, #192 ; 0xc0\n", - " 6c8: 23c2 movs r3, #194 ; 0xc2\n", - " 6ca: 23c4 movs r3, #196 ; 0xc4\n", - " 6cc: 23c6 movs r3, #198 ; 0xc6\n", - " 6ce: 23c8 movs r3, #200 ; 0xc8\n", - " 6d0: 23ca movs r3, #202 ; 0xca\n", - " 6d2: 23cc movs r3, #204 ; 0xcc\n", - " 6d4: 23ce movs r3, #206 ; 0xce\n", - " 6d6: 23d0 movs r3, #208 ; 0xd0\n", - " 6d8: 23d2 movs r3, #210 ; 0xd2\n", - " 6da: 23d4 movs r3, #212 ; 0xd4\n", - " 6dc: 23d6 movs r3, #214 ; 0xd6\n", - " 6de: 23d8 movs r3, #216 ; 0xd8\n", - " 6e0: 23da movs r3, #218 ; 0xda\n", - " 6e2: 23dc movs r3, #220 ; 0xdc\n", - " 6e4: 23de movs r3, #222 ; 0xde\n", - " 6e6: 23e0 movs r3, #224 ; 0xe0\n", - " 6e8: 23e2 movs r3, #226 ; 0xe2\n", - " 6ea: 23e4 movs r3, #228 ; 0xe4\n", - " 6ec: 23e6 movs r3, #230 ; 0xe6\n", - " 6ee: 23e8 movs r3, #232 ; 0xe8\n", - " 6f0: 23ea movs r3, #234 ; 0xea\n", - " 6f2: 23ec movs r3, #236 ; 0xec\n", - " 6f4: 23ee movs r3, #238 ; 0xee\n", - " 6f6: 23f0 movs r3, #240 ; 0xf0\n", - " 6f8: 23f2 movs r3, #242 ; 0xf2\n", - " 6fa: 23f4 movs r3, #244 ; 0xf4\n", - " 6fc: 23f6 movs r3, #246 ; 0xf6\n", - " 6fe: 23f8 movs r3, #248 ; 0xf8\n", - " 700: 23fa movs r3, #250 ; 0xfa\n", - " 702: 23fc movs r3, #252 ; 0xfc\n", - " 704: 23fe movs r3, #254 ; 0xfe\n", - " 706: 2300 movs r3, #0\n", - " 708: 2302 movs r3, #2\n", - " 70a: 2304 movs r3, #4\n", - " 70c: 2306 movs r3, #6\n", - " 70e: 2308 movs r3, #8\n", - " 710: 230a movs r3, #10\n", - " 712: 230c movs r3, #12\n", - " 714: 230e movs r3, #14\n", - " 716: 2310 movs r3, #16\n", - " 718: 2312 movs r3, #18\n", - " 71a: 2314 movs r3, #20\n", - " 71c: 2316 movs r3, #22\n", - " 71e: 2318 movs r3, #24\n", - " 720: 231a movs r3, #26\n", - " 722: 231c movs r3, #28\n", - " 724: 231e movs r3, #30\n", - " 726: 2320 movs r3, #32\n", - " 728: 2322 movs r3, #34 ; 0x22\n", - " 72a: 2324 movs r3, #36 ; 0x24\n", - " 72c: 2326 movs r3, #38 ; 0x26\n", - " 72e: 2328 movs r3, #40 ; 0x28\n", - " 730: 232a movs r3, #42 ; 0x2a\n", - " 732: 232c movs r3, #44 ; 0x2c\n", - " 734: 232e movs r3, #46 ; 0x2e\n", - " 736: 2330 movs r3, #48 ; 0x30\n", - " 738: 2332 movs r3, #50 ; 0x32\n", - " 73a: 2334 movs r3, #52 ; 0x34\n", - " 73c: 2336 movs r3, #54 ; 0x36\n", - " 73e: 2338 movs r3, #56 ; 0x38\n", - " 740: 233a movs r3, #58 ; 0x3a\n", - " 742: 233c movs r3, #60 ; 0x3c\n", - " 744: 233e movs r3, #62 ; 0x3e\n", - " 746: 2340 movs r3, #64 ; 0x40\n", - " 748: 2342 movs r3, #66 ; 0x42\n", - " 74a: 2344 movs r3, #68 ; 0x44\n", - " 74c: 2346 movs r3, #70 ; 0x46\n", - " 74e: 2348 movs r3, #72 ; 0x48\n", - " 750: 234a movs r3, #74 ; 0x4a\n", - " 752: 234c movs r3, #76 ; 0x4c\n", - " 754: 234e movs r3, #78 ; 0x4e\n", - " 756: 2350 movs r3, #80 ; 0x50\n", - " 758: 2352 movs r3, #82 ; 0x52\n", - " 75a: 2354 movs r3, #84 ; 0x54\n", - " 75c: 2356 movs r3, #86 ; 0x56\n", - " 75e: 2358 movs r3, #88 ; 0x58\n", - " 760: 235a movs r3, #90 ; 0x5a\n", - " 762: 235c movs r3, #92 ; 0x5c\n", - " 764: 235e movs r3, #94 ; 0x5e\n", - " 766: 2360 movs r3, #96 ; 0x60\n", - " 768: 2362 movs r3, #98 ; 0x62\n", - " 76a: 2364 movs r3, #100 ; 0x64\n", - " 76c: 2366 movs r3, #102 ; 0x66\n", - " 76e: 2368 movs r3, #104 ; 0x68\n", - " 770: 236a movs r3, #106 ; 0x6a\n", - " 772: 236c movs r3, #108 ; 0x6c\n", - " 774: 236e movs r3, #110 ; 0x6e\n", - " 776: 2370 movs r3, #112 ; 0x70\n", - " 778: 2372 movs r3, #114 ; 0x72\n", - " 77a: 2374 movs r3, #116 ; 0x74\n", - " 77c: 2376 movs r3, #118 ; 0x76\n", - " 77e: 2378 movs r3, #120 ; 0x78\n", - " 780: 237a movs r3, #122 ; 0x7a\n", - " 782: 237c movs r3, #124 ; 0x7c\n", - " 784: 237e movs r3, #126 ; 0x7e\n", - " 786: 2380 movs r3, #128 ; 0x80\n", - " 788: 2382 movs r3, #130 ; 0x82\n", - " 78a: 2384 movs r3, #132 ; 0x84\n", - " 78c: 2386 movs r3, #134 ; 0x86\n", - " 78e: 2388 movs r3, #136 ; 0x88\n", - " 790: 238a movs r3, #138 ; 0x8a\n", - " 792: 238c movs r3, #140 ; 0x8c\n", - " 794: 238e movs r3, #142 ; 0x8e\n", - " 796: 2390 movs r3, #144 ; 0x90\n", - " 798: 2392 movs r3, #146 ; 0x92\n", - " 79a: 2394 movs r3, #148 ; 0x94\n", - " 79c: 2396 movs r3, #150 ; 0x96\n", - " 79e: 2398 movs r3, #152 ; 0x98\n", - " 7a0: 239a movs r3, #154 ; 0x9a\n", - " 7a2: 239c movs r3, #156 ; 0x9c\n", - " 7a4: 239e movs r3, #158 ; 0x9e\n", - " 7a6: 23a0 movs r3, #160 ; 0xa0\n", - " 7a8: 23a2 movs r3, #162 ; 0xa2\n", - " 7aa: 23a4 movs r3, #164 ; 0xa4\n", - " 7ac: 23a6 movs r3, #166 ; 0xa6\n", - " 7ae: 23a8 movs r3, #168 ; 0xa8\n", - " 7b0: 23aa movs r3, #170 ; 0xaa\n", - " 7b2: 23ac movs r3, #172 ; 0xac\n", - " 7b4: 23ae movs r3, #174 ; 0xae\n", - " 7b6: 23b0 movs r3, #176 ; 0xb0\n", - " 7b8: 23b2 movs r3, #178 ; 0xb2\n", - " 7ba: 23b4 movs r3, #180 ; 0xb4\n", - " 7bc: 23b6 movs r3, #182 ; 0xb6\n", - " 7be: 23b8 movs r3, #184 ; 0xb8\n", - " 7c0: 23ba movs r3, #186 ; 0xba\n", - " 7c2: 23bc movs r3, #188 ; 0xbc\n", - " 7c4: 23be movs r3, #190 ; 0xbe\n", - " 7c6: 23c0 movs r3, #192 ; 0xc0\n", - " 7c8: 23c2 movs r3, #194 ; 0xc2\n", - " 7ca: 23c4 movs r3, #196 ; 0xc4\n", - " 7cc: 23c6 movs r3, #198 ; 0xc6\n", - " 7ce: 23c8 movs r3, #200 ; 0xc8\n", - " 7d0: 23ca movs r3, #202 ; 0xca\n", - " 7d2: 23cc movs r3, #204 ; 0xcc\n", - " 7d4: 23ce movs r3, #206 ; 0xce\n", - " 7d6: 23d0 movs r3, #208 ; 0xd0\n", - " 7d8: 23d2 movs r3, #210 ; 0xd2\n", - " 7da: 23d4 movs r3, #212 ; 0xd4\n", - " 7dc: 23d6 movs r3, #214 ; 0xd6\n", - " 7de: 23d8 movs r3, #216 ; 0xd8\n", - " 7e0: 23da movs r3, #218 ; 0xda\n", - " 7e2: 23dc movs r3, #220 ; 0xdc\n", - " 7e4: 23de movs r3, #222 ; 0xde\n", - " 7e6: 23e0 movs r3, #224 ; 0xe0\n", - " 7e8: 23e2 movs r3, #226 ; 0xe2\n", - " 7ea: 23e4 movs r3, #228 ; 0xe4\n", - " 7ec: 23e6 movs r3, #230 ; 0xe6\n", - " 7ee: 23e8 movs r3, #232 ; 0xe8\n", - " 7f0: 23ea movs r3, #234 ; 0xea\n", - " 7f2: 23ec movs r3, #236 ; 0xec\n", - " 7f4: 23ee movs r3, #238 ; 0xee\n", - " 7f6: 23f0 movs r3, #240 ; 0xf0\n", - " 7f8: 23f2 movs r3, #242 ; 0xf2\n", - " 7fa: 23f4 movs r3, #244 ; 0xf4\n", - " 7fc: 23f6 movs r3, #246 ; 0xf6\n", - " 7fe: 23f8 movs r3, #248 ; 0xf8\n", - " 800: 23fa movs r3, #250 ; 0xfa\n", - " 802: 23fc movs r3, #252 ; 0xfc\n", - " 804: 23fe movs r3, #254 ; 0xfe\n", - " 806: 2300 movs r3, #0\n", - " 808: 4611 mov r1, r2\n", - nullptr -}; -const char* const MixedBranch32Results[] = { - " 0: f000 bc03 b.w 80a <MixedBranch32+0x80a>\n", - " 4: 2300 movs r3, #0\n", - " 6: 2302 movs r3, #2\n", - " 8: 2304 movs r3, #4\n", - " a: 2306 movs r3, #6\n", - " c: 2308 movs r3, #8\n", - " e: 230a movs r3, #10\n", - " 10: 230c movs r3, #12\n", - " 12: 230e movs r3, #14\n", - " 14: 2310 movs r3, #16\n", - " 16: 2312 movs r3, #18\n", - " 18: 2314 movs r3, #20\n", - " 1a: 2316 movs r3, #22\n", - " 1c: 2318 movs r3, #24\n", - " 1e: 231a movs r3, #26\n", - " 20: 231c movs r3, #28\n", - " 22: 231e movs r3, #30\n", - " 24: 2320 movs r3, #32\n", - " 26: 2322 movs r3, #34 ; 0x22\n", - " 28: 2324 movs r3, #36 ; 0x24\n", - " 2a: 2326 movs r3, #38 ; 0x26\n", - " 2c: 2328 movs r3, #40 ; 0x28\n", - " 2e: 232a movs r3, #42 ; 0x2a\n", - " 30: 232c movs r3, #44 ; 0x2c\n", - " 32: 232e movs r3, #46 ; 0x2e\n", - " 34: 2330 movs r3, #48 ; 0x30\n", - " 36: 2332 movs r3, #50 ; 0x32\n", - " 38: 2334 movs r3, #52 ; 0x34\n", - " 3a: 2336 movs r3, #54 ; 0x36\n", - " 3c: 2338 movs r3, #56 ; 0x38\n", - " 3e: 233a movs r3, #58 ; 0x3a\n", - " 40: 233c movs r3, #60 ; 0x3c\n", - " 42: 233e movs r3, #62 ; 0x3e\n", - " 44: 2340 movs r3, #64 ; 0x40\n", - " 46: 2342 movs r3, #66 ; 0x42\n", - " 48: 2344 movs r3, #68 ; 0x44\n", - " 4a: 2346 movs r3, #70 ; 0x46\n", - " 4c: 2348 movs r3, #72 ; 0x48\n", - " 4e: 234a movs r3, #74 ; 0x4a\n", - " 50: 234c movs r3, #76 ; 0x4c\n", - " 52: 234e movs r3, #78 ; 0x4e\n", - " 54: 2350 movs r3, #80 ; 0x50\n", - " 56: 2352 movs r3, #82 ; 0x52\n", - " 58: 2354 movs r3, #84 ; 0x54\n", - " 5a: 2356 movs r3, #86 ; 0x56\n", - " 5c: 2358 movs r3, #88 ; 0x58\n", - " 5e: 235a movs r3, #90 ; 0x5a\n", - " 60: 235c movs r3, #92 ; 0x5c\n", - " 62: 235e movs r3, #94 ; 0x5e\n", - " 64: 2360 movs r3, #96 ; 0x60\n", - " 66: 2362 movs r3, #98 ; 0x62\n", - " 68: 2364 movs r3, #100 ; 0x64\n", - " 6a: 2366 movs r3, #102 ; 0x66\n", - " 6c: 2368 movs r3, #104 ; 0x68\n", - " 6e: 236a movs r3, #106 ; 0x6a\n", - " 70: 236c movs r3, #108 ; 0x6c\n", - " 72: 236e movs r3, #110 ; 0x6e\n", - " 74: 2370 movs r3, #112 ; 0x70\n", - " 76: 2372 movs r3, #114 ; 0x72\n", - " 78: 2374 movs r3, #116 ; 0x74\n", - " 7a: 2376 movs r3, #118 ; 0x76\n", - " 7c: 2378 movs r3, #120 ; 0x78\n", - " 7e: 237a movs r3, #122 ; 0x7a\n", - " 80: 237c movs r3, #124 ; 0x7c\n", - " 82: 237e movs r3, #126 ; 0x7e\n", - " 84: 2380 movs r3, #128 ; 0x80\n", - " 86: 2382 movs r3, #130 ; 0x82\n", - " 88: 2384 movs r3, #132 ; 0x84\n", - " 8a: 2386 movs r3, #134 ; 0x86\n", - " 8c: 2388 movs r3, #136 ; 0x88\n", - " 8e: 238a movs r3, #138 ; 0x8a\n", - " 90: 238c movs r3, #140 ; 0x8c\n", - " 92: 238e movs r3, #142 ; 0x8e\n", - " 94: 2390 movs r3, #144 ; 0x90\n", - " 96: 2392 movs r3, #146 ; 0x92\n", - " 98: 2394 movs r3, #148 ; 0x94\n", - " 9a: 2396 movs r3, #150 ; 0x96\n", - " 9c: 2398 movs r3, #152 ; 0x98\n", - " 9e: 239a movs r3, #154 ; 0x9a\n", - " a0: 239c movs r3, #156 ; 0x9c\n", - " a2: 239e movs r3, #158 ; 0x9e\n", - " a4: 23a0 movs r3, #160 ; 0xa0\n", - " a6: 23a2 movs r3, #162 ; 0xa2\n", - " a8: 23a4 movs r3, #164 ; 0xa4\n", - " aa: 23a6 movs r3, #166 ; 0xa6\n", - " ac: 23a8 movs r3, #168 ; 0xa8\n", - " ae: 23aa movs r3, #170 ; 0xaa\n", - " b0: 23ac movs r3, #172 ; 0xac\n", - " b2: 23ae movs r3, #174 ; 0xae\n", - " b4: 23b0 movs r3, #176 ; 0xb0\n", - " b6: 23b2 movs r3, #178 ; 0xb2\n", - " b8: 23b4 movs r3, #180 ; 0xb4\n", - " ba: 23b6 movs r3, #182 ; 0xb6\n", - " bc: 23b8 movs r3, #184 ; 0xb8\n", - " be: 23ba movs r3, #186 ; 0xba\n", - " c0: 23bc movs r3, #188 ; 0xbc\n", - " c2: 23be movs r3, #190 ; 0xbe\n", - " c4: 23c0 movs r3, #192 ; 0xc0\n", - " c6: 23c2 movs r3, #194 ; 0xc2\n", - " c8: 23c4 movs r3, #196 ; 0xc4\n", - " ca: 23c6 movs r3, #198 ; 0xc6\n", - " cc: 23c8 movs r3, #200 ; 0xc8\n", - " ce: 23ca movs r3, #202 ; 0xca\n", - " d0: 23cc movs r3, #204 ; 0xcc\n", - " d2: 23ce movs r3, #206 ; 0xce\n", - " d4: 23d0 movs r3, #208 ; 0xd0\n", - " d6: 23d2 movs r3, #210 ; 0xd2\n", - " d8: 23d4 movs r3, #212 ; 0xd4\n", - " da: 23d6 movs r3, #214 ; 0xd6\n", - " dc: 23d8 movs r3, #216 ; 0xd8\n", - " de: 23da movs r3, #218 ; 0xda\n", - " e0: 23dc movs r3, #220 ; 0xdc\n", - " e2: 23de movs r3, #222 ; 0xde\n", - " e4: 23e0 movs r3, #224 ; 0xe0\n", - " e6: 23e2 movs r3, #226 ; 0xe2\n", - " e8: 23e4 movs r3, #228 ; 0xe4\n", - " ea: 23e6 movs r3, #230 ; 0xe6\n", - " ec: 23e8 movs r3, #232 ; 0xe8\n", - " ee: 23ea movs r3, #234 ; 0xea\n", - " f0: 23ec movs r3, #236 ; 0xec\n", - " f2: 23ee movs r3, #238 ; 0xee\n", - " f4: 23f0 movs r3, #240 ; 0xf0\n", - " f6: 23f2 movs r3, #242 ; 0xf2\n", - " f8: 23f4 movs r3, #244 ; 0xf4\n", - " fa: 23f6 movs r3, #246 ; 0xf6\n", - " fc: 23f8 movs r3, #248 ; 0xf8\n", - " fe: 23fa movs r3, #250 ; 0xfa\n", - " 100: 23fc movs r3, #252 ; 0xfc\n", - " 102: 23fe movs r3, #254 ; 0xfe\n", - " 104: 2300 movs r3, #0\n", - " 106: 2302 movs r3, #2\n", - " 108: 2304 movs r3, #4\n", - " 10a: 2306 movs r3, #6\n", - " 10c: 2308 movs r3, #8\n", - " 10e: 230a movs r3, #10\n", - " 110: 230c movs r3, #12\n", - " 112: 230e movs r3, #14\n", - " 114: 2310 movs r3, #16\n", - " 116: 2312 movs r3, #18\n", - " 118: 2314 movs r3, #20\n", - " 11a: 2316 movs r3, #22\n", - " 11c: 2318 movs r3, #24\n", - " 11e: 231a movs r3, #26\n", - " 120: 231c movs r3, #28\n", - " 122: 231e movs r3, #30\n", - " 124: 2320 movs r3, #32\n", - " 126: 2322 movs r3, #34 ; 0x22\n", - " 128: 2324 movs r3, #36 ; 0x24\n", - " 12a: 2326 movs r3, #38 ; 0x26\n", - " 12c: 2328 movs r3, #40 ; 0x28\n", - " 12e: 232a movs r3, #42 ; 0x2a\n", - " 130: 232c movs r3, #44 ; 0x2c\n", - " 132: 232e movs r3, #46 ; 0x2e\n", - " 134: 2330 movs r3, #48 ; 0x30\n", - " 136: 2332 movs r3, #50 ; 0x32\n", - " 138: 2334 movs r3, #52 ; 0x34\n", - " 13a: 2336 movs r3, #54 ; 0x36\n", - " 13c: 2338 movs r3, #56 ; 0x38\n", - " 13e: 233a movs r3, #58 ; 0x3a\n", - " 140: 233c movs r3, #60 ; 0x3c\n", - " 142: 233e movs r3, #62 ; 0x3e\n", - " 144: 2340 movs r3, #64 ; 0x40\n", - " 146: 2342 movs r3, #66 ; 0x42\n", - " 148: 2344 movs r3, #68 ; 0x44\n", - " 14a: 2346 movs r3, #70 ; 0x46\n", - " 14c: 2348 movs r3, #72 ; 0x48\n", - " 14e: 234a movs r3, #74 ; 0x4a\n", - " 150: 234c movs r3, #76 ; 0x4c\n", - " 152: 234e movs r3, #78 ; 0x4e\n", - " 154: 2350 movs r3, #80 ; 0x50\n", - " 156: 2352 movs r3, #82 ; 0x52\n", - " 158: 2354 movs r3, #84 ; 0x54\n", - " 15a: 2356 movs r3, #86 ; 0x56\n", - " 15c: 2358 movs r3, #88 ; 0x58\n", - " 15e: 235a movs r3, #90 ; 0x5a\n", - " 160: 235c movs r3, #92 ; 0x5c\n", - " 162: 235e movs r3, #94 ; 0x5e\n", - " 164: 2360 movs r3, #96 ; 0x60\n", - " 166: 2362 movs r3, #98 ; 0x62\n", - " 168: 2364 movs r3, #100 ; 0x64\n", - " 16a: 2366 movs r3, #102 ; 0x66\n", - " 16c: 2368 movs r3, #104 ; 0x68\n", - " 16e: 236a movs r3, #106 ; 0x6a\n", - " 170: 236c movs r3, #108 ; 0x6c\n", - " 172: 236e movs r3, #110 ; 0x6e\n", - " 174: 2370 movs r3, #112 ; 0x70\n", - " 176: 2372 movs r3, #114 ; 0x72\n", - " 178: 2374 movs r3, #116 ; 0x74\n", - " 17a: 2376 movs r3, #118 ; 0x76\n", - " 17c: 2378 movs r3, #120 ; 0x78\n", - " 17e: 237a movs r3, #122 ; 0x7a\n", - " 180: 237c movs r3, #124 ; 0x7c\n", - " 182: 237e movs r3, #126 ; 0x7e\n", - " 184: 2380 movs r3, #128 ; 0x80\n", - " 186: 2382 movs r3, #130 ; 0x82\n", - " 188: 2384 movs r3, #132 ; 0x84\n", - " 18a: 2386 movs r3, #134 ; 0x86\n", - " 18c: 2388 movs r3, #136 ; 0x88\n", - " 18e: 238a movs r3, #138 ; 0x8a\n", - " 190: 238c movs r3, #140 ; 0x8c\n", - " 192: 238e movs r3, #142 ; 0x8e\n", - " 194: 2390 movs r3, #144 ; 0x90\n", - " 196: 2392 movs r3, #146 ; 0x92\n", - " 198: 2394 movs r3, #148 ; 0x94\n", - " 19a: 2396 movs r3, #150 ; 0x96\n", - " 19c: 2398 movs r3, #152 ; 0x98\n", - " 19e: 239a movs r3, #154 ; 0x9a\n", - " 1a0: 239c movs r3, #156 ; 0x9c\n", - " 1a2: 239e movs r3, #158 ; 0x9e\n", - " 1a4: 23a0 movs r3, #160 ; 0xa0\n", - " 1a6: 23a2 movs r3, #162 ; 0xa2\n", - " 1a8: 23a4 movs r3, #164 ; 0xa4\n", - " 1aa: 23a6 movs r3, #166 ; 0xa6\n", - " 1ac: 23a8 movs r3, #168 ; 0xa8\n", - " 1ae: 23aa movs r3, #170 ; 0xaa\n", - " 1b0: 23ac movs r3, #172 ; 0xac\n", - " 1b2: 23ae movs r3, #174 ; 0xae\n", - " 1b4: 23b0 movs r3, #176 ; 0xb0\n", - " 1b6: 23b2 movs r3, #178 ; 0xb2\n", - " 1b8: 23b4 movs r3, #180 ; 0xb4\n", - " 1ba: 23b6 movs r3, #182 ; 0xb6\n", - " 1bc: 23b8 movs r3, #184 ; 0xb8\n", - " 1be: 23ba movs r3, #186 ; 0xba\n", - " 1c0: 23bc movs r3, #188 ; 0xbc\n", - " 1c2: 23be movs r3, #190 ; 0xbe\n", - " 1c4: 23c0 movs r3, #192 ; 0xc0\n", - " 1c6: 23c2 movs r3, #194 ; 0xc2\n", - " 1c8: 23c4 movs r3, #196 ; 0xc4\n", - " 1ca: 23c6 movs r3, #198 ; 0xc6\n", - " 1cc: 23c8 movs r3, #200 ; 0xc8\n", - " 1ce: 23ca movs r3, #202 ; 0xca\n", - " 1d0: 23cc movs r3, #204 ; 0xcc\n", - " 1d2: 23ce movs r3, #206 ; 0xce\n", - " 1d4: 23d0 movs r3, #208 ; 0xd0\n", - " 1d6: 23d2 movs r3, #210 ; 0xd2\n", - " 1d8: 23d4 movs r3, #212 ; 0xd4\n", - " 1da: 23d6 movs r3, #214 ; 0xd6\n", - " 1dc: 23d8 movs r3, #216 ; 0xd8\n", - " 1de: 23da movs r3, #218 ; 0xda\n", - " 1e0: 23dc movs r3, #220 ; 0xdc\n", - " 1e2: 23de movs r3, #222 ; 0xde\n", - " 1e4: 23e0 movs r3, #224 ; 0xe0\n", - " 1e6: 23e2 movs r3, #226 ; 0xe2\n", - " 1e8: 23e4 movs r3, #228 ; 0xe4\n", - " 1ea: 23e6 movs r3, #230 ; 0xe6\n", - " 1ec: 23e8 movs r3, #232 ; 0xe8\n", - " 1ee: 23ea movs r3, #234 ; 0xea\n", - " 1f0: 23ec movs r3, #236 ; 0xec\n", - " 1f2: 23ee movs r3, #238 ; 0xee\n", - " 1f4: 23f0 movs r3, #240 ; 0xf0\n", - " 1f6: 23f2 movs r3, #242 ; 0xf2\n", - " 1f8: 23f4 movs r3, #244 ; 0xf4\n", - " 1fa: 23f6 movs r3, #246 ; 0xf6\n", - " 1fc: 23f8 movs r3, #248 ; 0xf8\n", - " 1fe: 23fa movs r3, #250 ; 0xfa\n", - " 200: 23fc movs r3, #252 ; 0xfc\n", - " 202: 23fe movs r3, #254 ; 0xfe\n", - " 204: 2300 movs r3, #0\n", - " 206: 2302 movs r3, #2\n", - " 208: 2304 movs r3, #4\n", - " 20a: 2306 movs r3, #6\n", - " 20c: 2308 movs r3, #8\n", - " 20e: 230a movs r3, #10\n", - " 210: 230c movs r3, #12\n", - " 212: 230e movs r3, #14\n", - " 214: 2310 movs r3, #16\n", - " 216: 2312 movs r3, #18\n", - " 218: 2314 movs r3, #20\n", - " 21a: 2316 movs r3, #22\n", - " 21c: 2318 movs r3, #24\n", - " 21e: 231a movs r3, #26\n", - " 220: 231c movs r3, #28\n", - " 222: 231e movs r3, #30\n", - " 224: 2320 movs r3, #32\n", - " 226: 2322 movs r3, #34 ; 0x22\n", - " 228: 2324 movs r3, #36 ; 0x24\n", - " 22a: 2326 movs r3, #38 ; 0x26\n", - " 22c: 2328 movs r3, #40 ; 0x28\n", - " 22e: 232a movs r3, #42 ; 0x2a\n", - " 230: 232c movs r3, #44 ; 0x2c\n", - " 232: 232e movs r3, #46 ; 0x2e\n", - " 234: 2330 movs r3, #48 ; 0x30\n", - " 236: 2332 movs r3, #50 ; 0x32\n", - " 238: 2334 movs r3, #52 ; 0x34\n", - " 23a: 2336 movs r3, #54 ; 0x36\n", - " 23c: 2338 movs r3, #56 ; 0x38\n", - " 23e: 233a movs r3, #58 ; 0x3a\n", - " 240: 233c movs r3, #60 ; 0x3c\n", - " 242: 233e movs r3, #62 ; 0x3e\n", - " 244: 2340 movs r3, #64 ; 0x40\n", - " 246: 2342 movs r3, #66 ; 0x42\n", - " 248: 2344 movs r3, #68 ; 0x44\n", - " 24a: 2346 movs r3, #70 ; 0x46\n", - " 24c: 2348 movs r3, #72 ; 0x48\n", - " 24e: 234a movs r3, #74 ; 0x4a\n", - " 250: 234c movs r3, #76 ; 0x4c\n", - " 252: 234e movs r3, #78 ; 0x4e\n", - " 254: 2350 movs r3, #80 ; 0x50\n", - " 256: 2352 movs r3, #82 ; 0x52\n", - " 258: 2354 movs r3, #84 ; 0x54\n", - " 25a: 2356 movs r3, #86 ; 0x56\n", - " 25c: 2358 movs r3, #88 ; 0x58\n", - " 25e: 235a movs r3, #90 ; 0x5a\n", - " 260: 235c movs r3, #92 ; 0x5c\n", - " 262: 235e movs r3, #94 ; 0x5e\n", - " 264: 2360 movs r3, #96 ; 0x60\n", - " 266: 2362 movs r3, #98 ; 0x62\n", - " 268: 2364 movs r3, #100 ; 0x64\n", - " 26a: 2366 movs r3, #102 ; 0x66\n", - " 26c: 2368 movs r3, #104 ; 0x68\n", - " 26e: 236a movs r3, #106 ; 0x6a\n", - " 270: 236c movs r3, #108 ; 0x6c\n", - " 272: 236e movs r3, #110 ; 0x6e\n", - " 274: 2370 movs r3, #112 ; 0x70\n", - " 276: 2372 movs r3, #114 ; 0x72\n", - " 278: 2374 movs r3, #116 ; 0x74\n", - " 27a: 2376 movs r3, #118 ; 0x76\n", - " 27c: 2378 movs r3, #120 ; 0x78\n", - " 27e: 237a movs r3, #122 ; 0x7a\n", - " 280: 237c movs r3, #124 ; 0x7c\n", - " 282: 237e movs r3, #126 ; 0x7e\n", - " 284: 2380 movs r3, #128 ; 0x80\n", - " 286: 2382 movs r3, #130 ; 0x82\n", - " 288: 2384 movs r3, #132 ; 0x84\n", - " 28a: 2386 movs r3, #134 ; 0x86\n", - " 28c: 2388 movs r3, #136 ; 0x88\n", - " 28e: 238a movs r3, #138 ; 0x8a\n", - " 290: 238c movs r3, #140 ; 0x8c\n", - " 292: 238e movs r3, #142 ; 0x8e\n", - " 294: 2390 movs r3, #144 ; 0x90\n", - " 296: 2392 movs r3, #146 ; 0x92\n", - " 298: 2394 movs r3, #148 ; 0x94\n", - " 29a: 2396 movs r3, #150 ; 0x96\n", - " 29c: 2398 movs r3, #152 ; 0x98\n", - " 29e: 239a movs r3, #154 ; 0x9a\n", - " 2a0: 239c movs r3, #156 ; 0x9c\n", - " 2a2: 239e movs r3, #158 ; 0x9e\n", - " 2a4: 23a0 movs r3, #160 ; 0xa0\n", - " 2a6: 23a2 movs r3, #162 ; 0xa2\n", - " 2a8: 23a4 movs r3, #164 ; 0xa4\n", - " 2aa: 23a6 movs r3, #166 ; 0xa6\n", - " 2ac: 23a8 movs r3, #168 ; 0xa8\n", - " 2ae: 23aa movs r3, #170 ; 0xaa\n", - " 2b0: 23ac movs r3, #172 ; 0xac\n", - " 2b2: 23ae movs r3, #174 ; 0xae\n", - " 2b4: 23b0 movs r3, #176 ; 0xb0\n", - " 2b6: 23b2 movs r3, #178 ; 0xb2\n", - " 2b8: 23b4 movs r3, #180 ; 0xb4\n", - " 2ba: 23b6 movs r3, #182 ; 0xb6\n", - " 2bc: 23b8 movs r3, #184 ; 0xb8\n", - " 2be: 23ba movs r3, #186 ; 0xba\n", - " 2c0: 23bc movs r3, #188 ; 0xbc\n", - " 2c2: 23be movs r3, #190 ; 0xbe\n", - " 2c4: 23c0 movs r3, #192 ; 0xc0\n", - " 2c6: 23c2 movs r3, #194 ; 0xc2\n", - " 2c8: 23c4 movs r3, #196 ; 0xc4\n", - " 2ca: 23c6 movs r3, #198 ; 0xc6\n", - " 2cc: 23c8 movs r3, #200 ; 0xc8\n", - " 2ce: 23ca movs r3, #202 ; 0xca\n", - " 2d0: 23cc movs r3, #204 ; 0xcc\n", - " 2d2: 23ce movs r3, #206 ; 0xce\n", - " 2d4: 23d0 movs r3, #208 ; 0xd0\n", - " 2d6: 23d2 movs r3, #210 ; 0xd2\n", - " 2d8: 23d4 movs r3, #212 ; 0xd4\n", - " 2da: 23d6 movs r3, #214 ; 0xd6\n", - " 2dc: 23d8 movs r3, #216 ; 0xd8\n", - " 2de: 23da movs r3, #218 ; 0xda\n", - " 2e0: 23dc movs r3, #220 ; 0xdc\n", - " 2e2: 23de movs r3, #222 ; 0xde\n", - " 2e4: 23e0 movs r3, #224 ; 0xe0\n", - " 2e6: 23e2 movs r3, #226 ; 0xe2\n", - " 2e8: 23e4 movs r3, #228 ; 0xe4\n", - " 2ea: 23e6 movs r3, #230 ; 0xe6\n", - " 2ec: 23e8 movs r3, #232 ; 0xe8\n", - " 2ee: 23ea movs r3, #234 ; 0xea\n", - " 2f0: 23ec movs r3, #236 ; 0xec\n", - " 2f2: 23ee movs r3, #238 ; 0xee\n", - " 2f4: 23f0 movs r3, #240 ; 0xf0\n", - " 2f6: 23f2 movs r3, #242 ; 0xf2\n", - " 2f8: 23f4 movs r3, #244 ; 0xf4\n", - " 2fa: 23f6 movs r3, #246 ; 0xf6\n", - " 2fc: 23f8 movs r3, #248 ; 0xf8\n", - " 2fe: 23fa movs r3, #250 ; 0xfa\n", - " 300: 23fc movs r3, #252 ; 0xfc\n", - " 302: 23fe movs r3, #254 ; 0xfe\n", - " 304: 2300 movs r3, #0\n", - " 306: 2302 movs r3, #2\n", - " 308: 2304 movs r3, #4\n", - " 30a: 2306 movs r3, #6\n", - " 30c: 2308 movs r3, #8\n", - " 30e: 230a movs r3, #10\n", - " 310: 230c movs r3, #12\n", - " 312: 230e movs r3, #14\n", - " 314: 2310 movs r3, #16\n", - " 316: 2312 movs r3, #18\n", - " 318: 2314 movs r3, #20\n", - " 31a: 2316 movs r3, #22\n", - " 31c: 2318 movs r3, #24\n", - " 31e: 231a movs r3, #26\n", - " 320: 231c movs r3, #28\n", - " 322: 231e movs r3, #30\n", - " 324: 2320 movs r3, #32\n", - " 326: 2322 movs r3, #34 ; 0x22\n", - " 328: 2324 movs r3, #36 ; 0x24\n", - " 32a: 2326 movs r3, #38 ; 0x26\n", - " 32c: 2328 movs r3, #40 ; 0x28\n", - " 32e: 232a movs r3, #42 ; 0x2a\n", - " 330: 232c movs r3, #44 ; 0x2c\n", - " 332: 232e movs r3, #46 ; 0x2e\n", - " 334: 2330 movs r3, #48 ; 0x30\n", - " 336: 2332 movs r3, #50 ; 0x32\n", - " 338: 2334 movs r3, #52 ; 0x34\n", - " 33a: 2336 movs r3, #54 ; 0x36\n", - " 33c: 2338 movs r3, #56 ; 0x38\n", - " 33e: 233a movs r3, #58 ; 0x3a\n", - " 340: 233c movs r3, #60 ; 0x3c\n", - " 342: 233e movs r3, #62 ; 0x3e\n", - " 344: 2340 movs r3, #64 ; 0x40\n", - " 346: 2342 movs r3, #66 ; 0x42\n", - " 348: 2344 movs r3, #68 ; 0x44\n", - " 34a: 2346 movs r3, #70 ; 0x46\n", - " 34c: 2348 movs r3, #72 ; 0x48\n", - " 34e: 234a movs r3, #74 ; 0x4a\n", - " 350: 234c movs r3, #76 ; 0x4c\n", - " 352: 234e movs r3, #78 ; 0x4e\n", - " 354: 2350 movs r3, #80 ; 0x50\n", - " 356: 2352 movs r3, #82 ; 0x52\n", - " 358: 2354 movs r3, #84 ; 0x54\n", - " 35a: 2356 movs r3, #86 ; 0x56\n", - " 35c: 2358 movs r3, #88 ; 0x58\n", - " 35e: 235a movs r3, #90 ; 0x5a\n", - " 360: 235c movs r3, #92 ; 0x5c\n", - " 362: 235e movs r3, #94 ; 0x5e\n", - " 364: 2360 movs r3, #96 ; 0x60\n", - " 366: 2362 movs r3, #98 ; 0x62\n", - " 368: 2364 movs r3, #100 ; 0x64\n", - " 36a: 2366 movs r3, #102 ; 0x66\n", - " 36c: 2368 movs r3, #104 ; 0x68\n", - " 36e: 236a movs r3, #106 ; 0x6a\n", - " 370: 236c movs r3, #108 ; 0x6c\n", - " 372: 236e movs r3, #110 ; 0x6e\n", - " 374: 2370 movs r3, #112 ; 0x70\n", - " 376: 2372 movs r3, #114 ; 0x72\n", - " 378: 2374 movs r3, #116 ; 0x74\n", - " 37a: 2376 movs r3, #118 ; 0x76\n", - " 37c: 2378 movs r3, #120 ; 0x78\n", - " 37e: 237a movs r3, #122 ; 0x7a\n", - " 380: 237c movs r3, #124 ; 0x7c\n", - " 382: 237e movs r3, #126 ; 0x7e\n", - " 384: 2380 movs r3, #128 ; 0x80\n", - " 386: 2382 movs r3, #130 ; 0x82\n", - " 388: 2384 movs r3, #132 ; 0x84\n", - " 38a: 2386 movs r3, #134 ; 0x86\n", - " 38c: 2388 movs r3, #136 ; 0x88\n", - " 38e: 238a movs r3, #138 ; 0x8a\n", - " 390: 238c movs r3, #140 ; 0x8c\n", - " 392: 238e movs r3, #142 ; 0x8e\n", - " 394: 2390 movs r3, #144 ; 0x90\n", - " 396: 2392 movs r3, #146 ; 0x92\n", - " 398: 2394 movs r3, #148 ; 0x94\n", - " 39a: 2396 movs r3, #150 ; 0x96\n", - " 39c: 2398 movs r3, #152 ; 0x98\n", - " 39e: 239a movs r3, #154 ; 0x9a\n", - " 3a0: 239c movs r3, #156 ; 0x9c\n", - " 3a2: 239e movs r3, #158 ; 0x9e\n", - " 3a4: 23a0 movs r3, #160 ; 0xa0\n", - " 3a6: 23a2 movs r3, #162 ; 0xa2\n", - " 3a8: 23a4 movs r3, #164 ; 0xa4\n", - " 3aa: 23a6 movs r3, #166 ; 0xa6\n", - " 3ac: 23a8 movs r3, #168 ; 0xa8\n", - " 3ae: 23aa movs r3, #170 ; 0xaa\n", - " 3b0: 23ac movs r3, #172 ; 0xac\n", - " 3b2: 23ae movs r3, #174 ; 0xae\n", - " 3b4: 23b0 movs r3, #176 ; 0xb0\n", - " 3b6: 23b2 movs r3, #178 ; 0xb2\n", - " 3b8: 23b4 movs r3, #180 ; 0xb4\n", - " 3ba: 23b6 movs r3, #182 ; 0xb6\n", - " 3bc: 23b8 movs r3, #184 ; 0xb8\n", - " 3be: 23ba movs r3, #186 ; 0xba\n", - " 3c0: 23bc movs r3, #188 ; 0xbc\n", - " 3c2: 23be movs r3, #190 ; 0xbe\n", - " 3c4: 23c0 movs r3, #192 ; 0xc0\n", - " 3c6: 23c2 movs r3, #194 ; 0xc2\n", - " 3c8: 23c4 movs r3, #196 ; 0xc4\n", - " 3ca: 23c6 movs r3, #198 ; 0xc6\n", - " 3cc: 23c8 movs r3, #200 ; 0xc8\n", - " 3ce: 23ca movs r3, #202 ; 0xca\n", - " 3d0: 23cc movs r3, #204 ; 0xcc\n", - " 3d2: 23ce movs r3, #206 ; 0xce\n", - " 3d4: 23d0 movs r3, #208 ; 0xd0\n", - " 3d6: 23d2 movs r3, #210 ; 0xd2\n", - " 3d8: 23d4 movs r3, #212 ; 0xd4\n", - " 3da: 23d6 movs r3, #214 ; 0xd6\n", - " 3dc: 23d8 movs r3, #216 ; 0xd8\n", - " 3de: 23da movs r3, #218 ; 0xda\n", - " 3e0: 23dc movs r3, #220 ; 0xdc\n", - " 3e2: 23de movs r3, #222 ; 0xde\n", - " 3e4: 23e0 movs r3, #224 ; 0xe0\n", - " 3e6: 23e2 movs r3, #226 ; 0xe2\n", - " 3e8: 23e4 movs r3, #228 ; 0xe4\n", - " 3ea: 23e6 movs r3, #230 ; 0xe6\n", - " 3ec: 23e8 movs r3, #232 ; 0xe8\n", - " 3ee: 23ea movs r3, #234 ; 0xea\n", - " 3f0: 23ec movs r3, #236 ; 0xec\n", - " 3f2: 23ee movs r3, #238 ; 0xee\n", - " 3f4: 23f0 movs r3, #240 ; 0xf0\n", - " 3f6: 23f2 movs r3, #242 ; 0xf2\n", - " 3f8: 23f4 movs r3, #244 ; 0xf4\n", - " 3fa: 23f6 movs r3, #246 ; 0xf6\n", - " 3fc: 23f8 movs r3, #248 ; 0xf8\n", - " 3fe: 23fa movs r3, #250 ; 0xfa\n", - " 400: 23fc movs r3, #252 ; 0xfc\n", - " 402: 23fe movs r3, #254 ; 0xfe\n", - " 404: 2300 movs r3, #0\n", - " 406: 2302 movs r3, #2\n", - " 408: 2304 movs r3, #4\n", - " 40a: 2306 movs r3, #6\n", - " 40c: 2308 movs r3, #8\n", - " 40e: 230a movs r3, #10\n", - " 410: 230c movs r3, #12\n", - " 412: 230e movs r3, #14\n", - " 414: 2310 movs r3, #16\n", - " 416: 2312 movs r3, #18\n", - " 418: 2314 movs r3, #20\n", - " 41a: 2316 movs r3, #22\n", - " 41c: 2318 movs r3, #24\n", - " 41e: 231a movs r3, #26\n", - " 420: 231c movs r3, #28\n", - " 422: 231e movs r3, #30\n", - " 424: 2320 movs r3, #32\n", - " 426: 2322 movs r3, #34 ; 0x22\n", - " 428: 2324 movs r3, #36 ; 0x24\n", - " 42a: 2326 movs r3, #38 ; 0x26\n", - " 42c: 2328 movs r3, #40 ; 0x28\n", - " 42e: 232a movs r3, #42 ; 0x2a\n", - " 430: 232c movs r3, #44 ; 0x2c\n", - " 432: 232e movs r3, #46 ; 0x2e\n", - " 434: 2330 movs r3, #48 ; 0x30\n", - " 436: 2332 movs r3, #50 ; 0x32\n", - " 438: 2334 movs r3, #52 ; 0x34\n", - " 43a: 2336 movs r3, #54 ; 0x36\n", - " 43c: 2338 movs r3, #56 ; 0x38\n", - " 43e: 233a movs r3, #58 ; 0x3a\n", - " 440: 233c movs r3, #60 ; 0x3c\n", - " 442: 233e movs r3, #62 ; 0x3e\n", - " 444: 2340 movs r3, #64 ; 0x40\n", - " 446: 2342 movs r3, #66 ; 0x42\n", - " 448: 2344 movs r3, #68 ; 0x44\n", - " 44a: 2346 movs r3, #70 ; 0x46\n", - " 44c: 2348 movs r3, #72 ; 0x48\n", - " 44e: 234a movs r3, #74 ; 0x4a\n", - " 450: 234c movs r3, #76 ; 0x4c\n", - " 452: 234e movs r3, #78 ; 0x4e\n", - " 454: 2350 movs r3, #80 ; 0x50\n", - " 456: 2352 movs r3, #82 ; 0x52\n", - " 458: 2354 movs r3, #84 ; 0x54\n", - " 45a: 2356 movs r3, #86 ; 0x56\n", - " 45c: 2358 movs r3, #88 ; 0x58\n", - " 45e: 235a movs r3, #90 ; 0x5a\n", - " 460: 235c movs r3, #92 ; 0x5c\n", - " 462: 235e movs r3, #94 ; 0x5e\n", - " 464: 2360 movs r3, #96 ; 0x60\n", - " 466: 2362 movs r3, #98 ; 0x62\n", - " 468: 2364 movs r3, #100 ; 0x64\n", - " 46a: 2366 movs r3, #102 ; 0x66\n", - " 46c: 2368 movs r3, #104 ; 0x68\n", - " 46e: 236a movs r3, #106 ; 0x6a\n", - " 470: 236c movs r3, #108 ; 0x6c\n", - " 472: 236e movs r3, #110 ; 0x6e\n", - " 474: 2370 movs r3, #112 ; 0x70\n", - " 476: 2372 movs r3, #114 ; 0x72\n", - " 478: 2374 movs r3, #116 ; 0x74\n", - " 47a: 2376 movs r3, #118 ; 0x76\n", - " 47c: 2378 movs r3, #120 ; 0x78\n", - " 47e: 237a movs r3, #122 ; 0x7a\n", - " 480: 237c movs r3, #124 ; 0x7c\n", - " 482: 237e movs r3, #126 ; 0x7e\n", - " 484: 2380 movs r3, #128 ; 0x80\n", - " 486: 2382 movs r3, #130 ; 0x82\n", - " 488: 2384 movs r3, #132 ; 0x84\n", - " 48a: 2386 movs r3, #134 ; 0x86\n", - " 48c: 2388 movs r3, #136 ; 0x88\n", - " 48e: 238a movs r3, #138 ; 0x8a\n", - " 490: 238c movs r3, #140 ; 0x8c\n", - " 492: 238e movs r3, #142 ; 0x8e\n", - " 494: 2390 movs r3, #144 ; 0x90\n", - " 496: 2392 movs r3, #146 ; 0x92\n", - " 498: 2394 movs r3, #148 ; 0x94\n", - " 49a: 2396 movs r3, #150 ; 0x96\n", - " 49c: 2398 movs r3, #152 ; 0x98\n", - " 49e: 239a movs r3, #154 ; 0x9a\n", - " 4a0: 239c movs r3, #156 ; 0x9c\n", - " 4a2: 239e movs r3, #158 ; 0x9e\n", - " 4a4: 23a0 movs r3, #160 ; 0xa0\n", - " 4a6: 23a2 movs r3, #162 ; 0xa2\n", - " 4a8: 23a4 movs r3, #164 ; 0xa4\n", - " 4aa: 23a6 movs r3, #166 ; 0xa6\n", - " 4ac: 23a8 movs r3, #168 ; 0xa8\n", - " 4ae: 23aa movs r3, #170 ; 0xaa\n", - " 4b0: 23ac movs r3, #172 ; 0xac\n", - " 4b2: 23ae movs r3, #174 ; 0xae\n", - " 4b4: 23b0 movs r3, #176 ; 0xb0\n", - " 4b6: 23b2 movs r3, #178 ; 0xb2\n", - " 4b8: 23b4 movs r3, #180 ; 0xb4\n", - " 4ba: 23b6 movs r3, #182 ; 0xb6\n", - " 4bc: 23b8 movs r3, #184 ; 0xb8\n", - " 4be: 23ba movs r3, #186 ; 0xba\n", - " 4c0: 23bc movs r3, #188 ; 0xbc\n", - " 4c2: 23be movs r3, #190 ; 0xbe\n", - " 4c4: 23c0 movs r3, #192 ; 0xc0\n", - " 4c6: 23c2 movs r3, #194 ; 0xc2\n", - " 4c8: 23c4 movs r3, #196 ; 0xc4\n", - " 4ca: 23c6 movs r3, #198 ; 0xc6\n", - " 4cc: 23c8 movs r3, #200 ; 0xc8\n", - " 4ce: 23ca movs r3, #202 ; 0xca\n", - " 4d0: 23cc movs r3, #204 ; 0xcc\n", - " 4d2: 23ce movs r3, #206 ; 0xce\n", - " 4d4: 23d0 movs r3, #208 ; 0xd0\n", - " 4d6: 23d2 movs r3, #210 ; 0xd2\n", - " 4d8: 23d4 movs r3, #212 ; 0xd4\n", - " 4da: 23d6 movs r3, #214 ; 0xd6\n", - " 4dc: 23d8 movs r3, #216 ; 0xd8\n", - " 4de: 23da movs r3, #218 ; 0xda\n", - " 4e0: 23dc movs r3, #220 ; 0xdc\n", - " 4e2: 23de movs r3, #222 ; 0xde\n", - " 4e4: 23e0 movs r3, #224 ; 0xe0\n", - " 4e6: 23e2 movs r3, #226 ; 0xe2\n", - " 4e8: 23e4 movs r3, #228 ; 0xe4\n", - " 4ea: 23e6 movs r3, #230 ; 0xe6\n", - " 4ec: 23e8 movs r3, #232 ; 0xe8\n", - " 4ee: 23ea movs r3, #234 ; 0xea\n", - " 4f0: 23ec movs r3, #236 ; 0xec\n", - " 4f2: 23ee movs r3, #238 ; 0xee\n", - " 4f4: 23f0 movs r3, #240 ; 0xf0\n", - " 4f6: 23f2 movs r3, #242 ; 0xf2\n", - " 4f8: 23f4 movs r3, #244 ; 0xf4\n", - " 4fa: 23f6 movs r3, #246 ; 0xf6\n", - " 4fc: 23f8 movs r3, #248 ; 0xf8\n", - " 4fe: 23fa movs r3, #250 ; 0xfa\n", - " 500: 23fc movs r3, #252 ; 0xfc\n", - " 502: 23fe movs r3, #254 ; 0xfe\n", - " 504: 2300 movs r3, #0\n", - " 506: 2302 movs r3, #2\n", - " 508: 2304 movs r3, #4\n", - " 50a: 2306 movs r3, #6\n", - " 50c: 2308 movs r3, #8\n", - " 50e: 230a movs r3, #10\n", - " 510: 230c movs r3, #12\n", - " 512: 230e movs r3, #14\n", - " 514: 2310 movs r3, #16\n", - " 516: 2312 movs r3, #18\n", - " 518: 2314 movs r3, #20\n", - " 51a: 2316 movs r3, #22\n", - " 51c: 2318 movs r3, #24\n", - " 51e: 231a movs r3, #26\n", - " 520: 231c movs r3, #28\n", - " 522: 231e movs r3, #30\n", - " 524: 2320 movs r3, #32\n", - " 526: 2322 movs r3, #34 ; 0x22\n", - " 528: 2324 movs r3, #36 ; 0x24\n", - " 52a: 2326 movs r3, #38 ; 0x26\n", - " 52c: 2328 movs r3, #40 ; 0x28\n", - " 52e: 232a movs r3, #42 ; 0x2a\n", - " 530: 232c movs r3, #44 ; 0x2c\n", - " 532: 232e movs r3, #46 ; 0x2e\n", - " 534: 2330 movs r3, #48 ; 0x30\n", - " 536: 2332 movs r3, #50 ; 0x32\n", - " 538: 2334 movs r3, #52 ; 0x34\n", - " 53a: 2336 movs r3, #54 ; 0x36\n", - " 53c: 2338 movs r3, #56 ; 0x38\n", - " 53e: 233a movs r3, #58 ; 0x3a\n", - " 540: 233c movs r3, #60 ; 0x3c\n", - " 542: 233e movs r3, #62 ; 0x3e\n", - " 544: 2340 movs r3, #64 ; 0x40\n", - " 546: 2342 movs r3, #66 ; 0x42\n", - " 548: 2344 movs r3, #68 ; 0x44\n", - " 54a: 2346 movs r3, #70 ; 0x46\n", - " 54c: 2348 movs r3, #72 ; 0x48\n", - " 54e: 234a movs r3, #74 ; 0x4a\n", - " 550: 234c movs r3, #76 ; 0x4c\n", - " 552: 234e movs r3, #78 ; 0x4e\n", - " 554: 2350 movs r3, #80 ; 0x50\n", - " 556: 2352 movs r3, #82 ; 0x52\n", - " 558: 2354 movs r3, #84 ; 0x54\n", - " 55a: 2356 movs r3, #86 ; 0x56\n", - " 55c: 2358 movs r3, #88 ; 0x58\n", - " 55e: 235a movs r3, #90 ; 0x5a\n", - " 560: 235c movs r3, #92 ; 0x5c\n", - " 562: 235e movs r3, #94 ; 0x5e\n", - " 564: 2360 movs r3, #96 ; 0x60\n", - " 566: 2362 movs r3, #98 ; 0x62\n", - " 568: 2364 movs r3, #100 ; 0x64\n", - " 56a: 2366 movs r3, #102 ; 0x66\n", - " 56c: 2368 movs r3, #104 ; 0x68\n", - " 56e: 236a movs r3, #106 ; 0x6a\n", - " 570: 236c movs r3, #108 ; 0x6c\n", - " 572: 236e movs r3, #110 ; 0x6e\n", - " 574: 2370 movs r3, #112 ; 0x70\n", - " 576: 2372 movs r3, #114 ; 0x72\n", - " 578: 2374 movs r3, #116 ; 0x74\n", - " 57a: 2376 movs r3, #118 ; 0x76\n", - " 57c: 2378 movs r3, #120 ; 0x78\n", - " 57e: 237a movs r3, #122 ; 0x7a\n", - " 580: 237c movs r3, #124 ; 0x7c\n", - " 582: 237e movs r3, #126 ; 0x7e\n", - " 584: 2380 movs r3, #128 ; 0x80\n", - " 586: 2382 movs r3, #130 ; 0x82\n", - " 588: 2384 movs r3, #132 ; 0x84\n", - " 58a: 2386 movs r3, #134 ; 0x86\n", - " 58c: 2388 movs r3, #136 ; 0x88\n", - " 58e: 238a movs r3, #138 ; 0x8a\n", - " 590: 238c movs r3, #140 ; 0x8c\n", - " 592: 238e movs r3, #142 ; 0x8e\n", - " 594: 2390 movs r3, #144 ; 0x90\n", - " 596: 2392 movs r3, #146 ; 0x92\n", - " 598: 2394 movs r3, #148 ; 0x94\n", - " 59a: 2396 movs r3, #150 ; 0x96\n", - " 59c: 2398 movs r3, #152 ; 0x98\n", - " 59e: 239a movs r3, #154 ; 0x9a\n", - " 5a0: 239c movs r3, #156 ; 0x9c\n", - " 5a2: 239e movs r3, #158 ; 0x9e\n", - " 5a4: 23a0 movs r3, #160 ; 0xa0\n", - " 5a6: 23a2 movs r3, #162 ; 0xa2\n", - " 5a8: 23a4 movs r3, #164 ; 0xa4\n", - " 5aa: 23a6 movs r3, #166 ; 0xa6\n", - " 5ac: 23a8 movs r3, #168 ; 0xa8\n", - " 5ae: 23aa movs r3, #170 ; 0xaa\n", - " 5b0: 23ac movs r3, #172 ; 0xac\n", - " 5b2: 23ae movs r3, #174 ; 0xae\n", - " 5b4: 23b0 movs r3, #176 ; 0xb0\n", - " 5b6: 23b2 movs r3, #178 ; 0xb2\n", - " 5b8: 23b4 movs r3, #180 ; 0xb4\n", - " 5ba: 23b6 movs r3, #182 ; 0xb6\n", - " 5bc: 23b8 movs r3, #184 ; 0xb8\n", - " 5be: 23ba movs r3, #186 ; 0xba\n", - " 5c0: 23bc movs r3, #188 ; 0xbc\n", - " 5c2: 23be movs r3, #190 ; 0xbe\n", - " 5c4: 23c0 movs r3, #192 ; 0xc0\n", - " 5c6: 23c2 movs r3, #194 ; 0xc2\n", - " 5c8: 23c4 movs r3, #196 ; 0xc4\n", - " 5ca: 23c6 movs r3, #198 ; 0xc6\n", - " 5cc: 23c8 movs r3, #200 ; 0xc8\n", - " 5ce: 23ca movs r3, #202 ; 0xca\n", - " 5d0: 23cc movs r3, #204 ; 0xcc\n", - " 5d2: 23ce movs r3, #206 ; 0xce\n", - " 5d4: 23d0 movs r3, #208 ; 0xd0\n", - " 5d6: 23d2 movs r3, #210 ; 0xd2\n", - " 5d8: 23d4 movs r3, #212 ; 0xd4\n", - " 5da: 23d6 movs r3, #214 ; 0xd6\n", - " 5dc: 23d8 movs r3, #216 ; 0xd8\n", - " 5de: 23da movs r3, #218 ; 0xda\n", - " 5e0: 23dc movs r3, #220 ; 0xdc\n", - " 5e2: 23de movs r3, #222 ; 0xde\n", - " 5e4: 23e0 movs r3, #224 ; 0xe0\n", - " 5e6: 23e2 movs r3, #226 ; 0xe2\n", - " 5e8: 23e4 movs r3, #228 ; 0xe4\n", - " 5ea: 23e6 movs r3, #230 ; 0xe6\n", - " 5ec: 23e8 movs r3, #232 ; 0xe8\n", - " 5ee: 23ea movs r3, #234 ; 0xea\n", - " 5f0: 23ec movs r3, #236 ; 0xec\n", - " 5f2: 23ee movs r3, #238 ; 0xee\n", - " 5f4: 23f0 movs r3, #240 ; 0xf0\n", - " 5f6: 23f2 movs r3, #242 ; 0xf2\n", - " 5f8: 23f4 movs r3, #244 ; 0xf4\n", - " 5fa: 23f6 movs r3, #246 ; 0xf6\n", - " 5fc: 23f8 movs r3, #248 ; 0xf8\n", - " 5fe: 23fa movs r3, #250 ; 0xfa\n", - " 600: 23fc movs r3, #252 ; 0xfc\n", - " 602: 23fe movs r3, #254 ; 0xfe\n", - " 604: 2300 movs r3, #0\n", - " 606: 2302 movs r3, #2\n", - " 608: 2304 movs r3, #4\n", - " 60a: 2306 movs r3, #6\n", - " 60c: 2308 movs r3, #8\n", - " 60e: 230a movs r3, #10\n", - " 610: 230c movs r3, #12\n", - " 612: 230e movs r3, #14\n", - " 614: 2310 movs r3, #16\n", - " 616: 2312 movs r3, #18\n", - " 618: 2314 movs r3, #20\n", - " 61a: 2316 movs r3, #22\n", - " 61c: 2318 movs r3, #24\n", - " 61e: 231a movs r3, #26\n", - " 620: 231c movs r3, #28\n", - " 622: 231e movs r3, #30\n", - " 624: 2320 movs r3, #32\n", - " 626: 2322 movs r3, #34 ; 0x22\n", - " 628: 2324 movs r3, #36 ; 0x24\n", - " 62a: 2326 movs r3, #38 ; 0x26\n", - " 62c: 2328 movs r3, #40 ; 0x28\n", - " 62e: 232a movs r3, #42 ; 0x2a\n", - " 630: 232c movs r3, #44 ; 0x2c\n", - " 632: 232e movs r3, #46 ; 0x2e\n", - " 634: 2330 movs r3, #48 ; 0x30\n", - " 636: 2332 movs r3, #50 ; 0x32\n", - " 638: 2334 movs r3, #52 ; 0x34\n", - " 63a: 2336 movs r3, #54 ; 0x36\n", - " 63c: 2338 movs r3, #56 ; 0x38\n", - " 63e: 233a movs r3, #58 ; 0x3a\n", - " 640: 233c movs r3, #60 ; 0x3c\n", - " 642: 233e movs r3, #62 ; 0x3e\n", - " 644: 2340 movs r3, #64 ; 0x40\n", - " 646: 2342 movs r3, #66 ; 0x42\n", - " 648: 2344 movs r3, #68 ; 0x44\n", - " 64a: 2346 movs r3, #70 ; 0x46\n", - " 64c: 2348 movs r3, #72 ; 0x48\n", - " 64e: 234a movs r3, #74 ; 0x4a\n", - " 650: 234c movs r3, #76 ; 0x4c\n", - " 652: 234e movs r3, #78 ; 0x4e\n", - " 654: 2350 movs r3, #80 ; 0x50\n", - " 656: 2352 movs r3, #82 ; 0x52\n", - " 658: 2354 movs r3, #84 ; 0x54\n", - " 65a: 2356 movs r3, #86 ; 0x56\n", - " 65c: 2358 movs r3, #88 ; 0x58\n", - " 65e: 235a movs r3, #90 ; 0x5a\n", - " 660: 235c movs r3, #92 ; 0x5c\n", - " 662: 235e movs r3, #94 ; 0x5e\n", - " 664: 2360 movs r3, #96 ; 0x60\n", - " 666: 2362 movs r3, #98 ; 0x62\n", - " 668: 2364 movs r3, #100 ; 0x64\n", - " 66a: 2366 movs r3, #102 ; 0x66\n", - " 66c: 2368 movs r3, #104 ; 0x68\n", - " 66e: 236a movs r3, #106 ; 0x6a\n", - " 670: 236c movs r3, #108 ; 0x6c\n", - " 672: 236e movs r3, #110 ; 0x6e\n", - " 674: 2370 movs r3, #112 ; 0x70\n", - " 676: 2372 movs r3, #114 ; 0x72\n", - " 678: 2374 movs r3, #116 ; 0x74\n", - " 67a: 2376 movs r3, #118 ; 0x76\n", - " 67c: 2378 movs r3, #120 ; 0x78\n", - " 67e: 237a movs r3, #122 ; 0x7a\n", - " 680: 237c movs r3, #124 ; 0x7c\n", - " 682: 237e movs r3, #126 ; 0x7e\n", - " 684: 2380 movs r3, #128 ; 0x80\n", - " 686: 2382 movs r3, #130 ; 0x82\n", - " 688: 2384 movs r3, #132 ; 0x84\n", - " 68a: 2386 movs r3, #134 ; 0x86\n", - " 68c: 2388 movs r3, #136 ; 0x88\n", - " 68e: 238a movs r3, #138 ; 0x8a\n", - " 690: 238c movs r3, #140 ; 0x8c\n", - " 692: 238e movs r3, #142 ; 0x8e\n", - " 694: 2390 movs r3, #144 ; 0x90\n", - " 696: 2392 movs r3, #146 ; 0x92\n", - " 698: 2394 movs r3, #148 ; 0x94\n", - " 69a: 2396 movs r3, #150 ; 0x96\n", - " 69c: 2398 movs r3, #152 ; 0x98\n", - " 69e: 239a movs r3, #154 ; 0x9a\n", - " 6a0: 239c movs r3, #156 ; 0x9c\n", - " 6a2: 239e movs r3, #158 ; 0x9e\n", - " 6a4: 23a0 movs r3, #160 ; 0xa0\n", - " 6a6: 23a2 movs r3, #162 ; 0xa2\n", - " 6a8: 23a4 movs r3, #164 ; 0xa4\n", - " 6aa: 23a6 movs r3, #166 ; 0xa6\n", - " 6ac: 23a8 movs r3, #168 ; 0xa8\n", - " 6ae: 23aa movs r3, #170 ; 0xaa\n", - " 6b0: 23ac movs r3, #172 ; 0xac\n", - " 6b2: 23ae movs r3, #174 ; 0xae\n", - " 6b4: 23b0 movs r3, #176 ; 0xb0\n", - " 6b6: 23b2 movs r3, #178 ; 0xb2\n", - " 6b8: 23b4 movs r3, #180 ; 0xb4\n", - " 6ba: 23b6 movs r3, #182 ; 0xb6\n", - " 6bc: 23b8 movs r3, #184 ; 0xb8\n", - " 6be: 23ba movs r3, #186 ; 0xba\n", - " 6c0: 23bc movs r3, #188 ; 0xbc\n", - " 6c2: 23be movs r3, #190 ; 0xbe\n", - " 6c4: 23c0 movs r3, #192 ; 0xc0\n", - " 6c6: 23c2 movs r3, #194 ; 0xc2\n", - " 6c8: 23c4 movs r3, #196 ; 0xc4\n", - " 6ca: 23c6 movs r3, #198 ; 0xc6\n", - " 6cc: 23c8 movs r3, #200 ; 0xc8\n", - " 6ce: 23ca movs r3, #202 ; 0xca\n", - " 6d0: 23cc movs r3, #204 ; 0xcc\n", - " 6d2: 23ce movs r3, #206 ; 0xce\n", - " 6d4: 23d0 movs r3, #208 ; 0xd0\n", - " 6d6: 23d2 movs r3, #210 ; 0xd2\n", - " 6d8: 23d4 movs r3, #212 ; 0xd4\n", - " 6da: 23d6 movs r3, #214 ; 0xd6\n", - " 6dc: 23d8 movs r3, #216 ; 0xd8\n", - " 6de: 23da movs r3, #218 ; 0xda\n", - " 6e0: 23dc movs r3, #220 ; 0xdc\n", - " 6e2: 23de movs r3, #222 ; 0xde\n", - " 6e4: 23e0 movs r3, #224 ; 0xe0\n", - " 6e6: 23e2 movs r3, #226 ; 0xe2\n", - " 6e8: 23e4 movs r3, #228 ; 0xe4\n", - " 6ea: 23e6 movs r3, #230 ; 0xe6\n", - " 6ec: 23e8 movs r3, #232 ; 0xe8\n", - " 6ee: 23ea movs r3, #234 ; 0xea\n", - " 6f0: 23ec movs r3, #236 ; 0xec\n", - " 6f2: 23ee movs r3, #238 ; 0xee\n", - " 6f4: 23f0 movs r3, #240 ; 0xf0\n", - " 6f6: 23f2 movs r3, #242 ; 0xf2\n", - " 6f8: 23f4 movs r3, #244 ; 0xf4\n", - " 6fa: 23f6 movs r3, #246 ; 0xf6\n", - " 6fc: 23f8 movs r3, #248 ; 0xf8\n", - " 6fe: 23fa movs r3, #250 ; 0xfa\n", - " 700: 23fc movs r3, #252 ; 0xfc\n", - " 702: 23fe movs r3, #254 ; 0xfe\n", - " 704: 2300 movs r3, #0\n", - " 706: 2302 movs r3, #2\n", - " 708: 2304 movs r3, #4\n", - " 70a: 2306 movs r3, #6\n", - " 70c: 2308 movs r3, #8\n", - " 70e: 230a movs r3, #10\n", - " 710: 230c movs r3, #12\n", - " 712: 230e movs r3, #14\n", - " 714: 2310 movs r3, #16\n", - " 716: 2312 movs r3, #18\n", - " 718: 2314 movs r3, #20\n", - " 71a: 2316 movs r3, #22\n", - " 71c: 2318 movs r3, #24\n", - " 71e: 231a movs r3, #26\n", - " 720: 231c movs r3, #28\n", - " 722: 231e movs r3, #30\n", - " 724: 2320 movs r3, #32\n", - " 726: 2322 movs r3, #34 ; 0x22\n", - " 728: 2324 movs r3, #36 ; 0x24\n", - " 72a: 2326 movs r3, #38 ; 0x26\n", - " 72c: 2328 movs r3, #40 ; 0x28\n", - " 72e: 232a movs r3, #42 ; 0x2a\n", - " 730: 232c movs r3, #44 ; 0x2c\n", - " 732: 232e movs r3, #46 ; 0x2e\n", - " 734: 2330 movs r3, #48 ; 0x30\n", - " 736: 2332 movs r3, #50 ; 0x32\n", - " 738: 2334 movs r3, #52 ; 0x34\n", - " 73a: 2336 movs r3, #54 ; 0x36\n", - " 73c: 2338 movs r3, #56 ; 0x38\n", - " 73e: 233a movs r3, #58 ; 0x3a\n", - " 740: 233c movs r3, #60 ; 0x3c\n", - " 742: 233e movs r3, #62 ; 0x3e\n", - " 744: 2340 movs r3, #64 ; 0x40\n", - " 746: 2342 movs r3, #66 ; 0x42\n", - " 748: 2344 movs r3, #68 ; 0x44\n", - " 74a: 2346 movs r3, #70 ; 0x46\n", - " 74c: 2348 movs r3, #72 ; 0x48\n", - " 74e: 234a movs r3, #74 ; 0x4a\n", - " 750: 234c movs r3, #76 ; 0x4c\n", - " 752: 234e movs r3, #78 ; 0x4e\n", - " 754: 2350 movs r3, #80 ; 0x50\n", - " 756: 2352 movs r3, #82 ; 0x52\n", - " 758: 2354 movs r3, #84 ; 0x54\n", - " 75a: 2356 movs r3, #86 ; 0x56\n", - " 75c: 2358 movs r3, #88 ; 0x58\n", - " 75e: 235a movs r3, #90 ; 0x5a\n", - " 760: 235c movs r3, #92 ; 0x5c\n", - " 762: 235e movs r3, #94 ; 0x5e\n", - " 764: 2360 movs r3, #96 ; 0x60\n", - " 766: 2362 movs r3, #98 ; 0x62\n", - " 768: 2364 movs r3, #100 ; 0x64\n", - " 76a: 2366 movs r3, #102 ; 0x66\n", - " 76c: 2368 movs r3, #104 ; 0x68\n", - " 76e: 236a movs r3, #106 ; 0x6a\n", - " 770: 236c movs r3, #108 ; 0x6c\n", - " 772: 236e movs r3, #110 ; 0x6e\n", - " 774: 2370 movs r3, #112 ; 0x70\n", - " 776: 2372 movs r3, #114 ; 0x72\n", - " 778: 2374 movs r3, #116 ; 0x74\n", - " 77a: 2376 movs r3, #118 ; 0x76\n", - " 77c: 2378 movs r3, #120 ; 0x78\n", - " 77e: 237a movs r3, #122 ; 0x7a\n", - " 780: 237c movs r3, #124 ; 0x7c\n", - " 782: 237e movs r3, #126 ; 0x7e\n", - " 784: 2380 movs r3, #128 ; 0x80\n", - " 786: 2382 movs r3, #130 ; 0x82\n", - " 788: 2384 movs r3, #132 ; 0x84\n", - " 78a: 2386 movs r3, #134 ; 0x86\n", - " 78c: 2388 movs r3, #136 ; 0x88\n", - " 78e: 238a movs r3, #138 ; 0x8a\n", - " 790: 238c movs r3, #140 ; 0x8c\n", - " 792: 238e movs r3, #142 ; 0x8e\n", - " 794: 2390 movs r3, #144 ; 0x90\n", - " 796: 2392 movs r3, #146 ; 0x92\n", - " 798: 2394 movs r3, #148 ; 0x94\n", - " 79a: 2396 movs r3, #150 ; 0x96\n", - " 79c: 2398 movs r3, #152 ; 0x98\n", - " 79e: 239a movs r3, #154 ; 0x9a\n", - " 7a0: 239c movs r3, #156 ; 0x9c\n", - " 7a2: 239e movs r3, #158 ; 0x9e\n", - " 7a4: 23a0 movs r3, #160 ; 0xa0\n", - " 7a6: 23a2 movs r3, #162 ; 0xa2\n", - " 7a8: 23a4 movs r3, #164 ; 0xa4\n", - " 7aa: 23a6 movs r3, #166 ; 0xa6\n", - " 7ac: 23a8 movs r3, #168 ; 0xa8\n", - " 7ae: 23aa movs r3, #170 ; 0xaa\n", - " 7b0: 23ac movs r3, #172 ; 0xac\n", - " 7b2: 23ae movs r3, #174 ; 0xae\n", - " 7b4: 23b0 movs r3, #176 ; 0xb0\n", - " 7b6: 23b2 movs r3, #178 ; 0xb2\n", - " 7b8: 23b4 movs r3, #180 ; 0xb4\n", - " 7ba: 23b6 movs r3, #182 ; 0xb6\n", - " 7bc: 23b8 movs r3, #184 ; 0xb8\n", - " 7be: 23ba movs r3, #186 ; 0xba\n", - " 7c0: 23bc movs r3, #188 ; 0xbc\n", - " 7c2: 23be movs r3, #190 ; 0xbe\n", - " 7c4: 23c0 movs r3, #192 ; 0xc0\n", - " 7c6: 23c2 movs r3, #194 ; 0xc2\n", - " 7c8: 23c4 movs r3, #196 ; 0xc4\n", - " 7ca: 23c6 movs r3, #198 ; 0xc6\n", - " 7cc: 23c8 movs r3, #200 ; 0xc8\n", - " 7ce: 23ca movs r3, #202 ; 0xca\n", - " 7d0: 23cc movs r3, #204 ; 0xcc\n", - " 7d2: 23ce movs r3, #206 ; 0xce\n", - " 7d4: 23d0 movs r3, #208 ; 0xd0\n", - " 7d6: 23d2 movs r3, #210 ; 0xd2\n", - " 7d8: 23d4 movs r3, #212 ; 0xd4\n", - " 7da: 23d6 movs r3, #214 ; 0xd6\n", - " 7dc: 23d8 movs r3, #216 ; 0xd8\n", - " 7de: 23da movs r3, #218 ; 0xda\n", - " 7e0: 23dc movs r3, #220 ; 0xdc\n", - " 7e2: 23de movs r3, #222 ; 0xde\n", - " 7e4: 23e0 movs r3, #224 ; 0xe0\n", - " 7e6: 23e2 movs r3, #226 ; 0xe2\n", - " 7e8: 23e4 movs r3, #228 ; 0xe4\n", - " 7ea: 23e6 movs r3, #230 ; 0xe6\n", - " 7ec: 23e8 movs r3, #232 ; 0xe8\n", - " 7ee: 23ea movs r3, #234 ; 0xea\n", - " 7f0: 23ec movs r3, #236 ; 0xec\n", - " 7f2: 23ee movs r3, #238 ; 0xee\n", - " 7f4: 23f0 movs r3, #240 ; 0xf0\n", - " 7f6: 23f2 movs r3, #242 ; 0xf2\n", - " 7f8: 23f4 movs r3, #244 ; 0xf4\n", - " 7fa: 23f6 movs r3, #246 ; 0xf6\n", - " 7fc: 23f8 movs r3, #248 ; 0xf8\n", - " 7fe: 23fa movs r3, #250 ; 0xfa\n", - " 800: 23fc movs r3, #252 ; 0xfc\n", - " 802: 23fe movs r3, #254 ; 0xfe\n", - " 804: 2300 movs r3, #0\n", - " 806: f7ff bbfd b.w 4 <MixedBranch32+0x4>\n", - " 80a: 4611 mov r1, r2\n", - nullptr -}; -const char* const ShiftsResults[] = { - " 0: 0148 lsls r0, r1, #5\n", - " 2: 0948 lsrs r0, r1, #5\n", - " 4: 1148 asrs r0, r1, #5\n", - " 6: 4088 lsls r0, r1\n", - " 8: 40c8 lsrs r0, r1\n", - " a: 4108 asrs r0, r1\n", - " c: 41c8 rors r0, r1\n", - " e: 0148 lsls r0, r1, #5\n", - " 10: 0948 lsrs r0, r1, #5\n", - " 12: 1148 asrs r0, r1, #5\n", - " 14: 4088 lsls r0, r1\n", - " 16: 40c8 lsrs r0, r1\n", - " 18: 4108 asrs r0, r1\n", - " 1a: 41c8 rors r0, r1\n", - " 1c: ea4f 1041 mov.w r0, r1, lsl #5\n", - " 20: ea4f 1051 mov.w r0, r1, lsr #5\n", - " 24: ea4f 1061 mov.w r0, r1, asr #5\n", - " 28: fa00 f001 lsl.w r0, r0, r1\n", - " 2c: fa20 f001 lsr.w r0, r0, r1\n", - " 30: fa40 f001 asr.w r0, r0, r1\n", - " 34: fa60 f001 ror.w r0, r0, r1\n", - " 38: ea4f 1071 mov.w r0, r1, ror #5\n", - " 3c: ea5f 1071 movs.w r0, r1, ror #5\n", - " 40: ea4f 1071 mov.w r0, r1, ror #5\n", - " 44: ea4f 1841 mov.w r8, r1, lsl #5\n", - " 48: ea4f 1058 mov.w r0, r8, lsr #5\n", - " 4c: ea4f 1861 mov.w r8, r1, asr #5\n", - " 50: ea4f 1078 mov.w r0, r8, ror #5\n", - " 54: fa01 f002 lsl.w r0, r1, r2\n", - " 58: fa21 f002 lsr.w r0, r1, r2\n", - " 5c: fa41 f002 asr.w r0, r1, r2\n", - " 60: fa61 f002 ror.w r0, r1, r2\n", - " 64: fa01 f802 lsl.w r8, r1, r2\n", - " 68: fa28 f002 lsr.w r0, r8, r2\n", - " 6c: fa41 f008 asr.w r0, r1, r8\n", - " 70: ea5f 1841 movs.w r8, r1, lsl #5\n", - " 74: ea5f 1058 movs.w r0, r8, lsr #5\n", - " 78: ea5f 1861 movs.w r8, r1, asr #5\n", - " 7c: ea5f 1078 movs.w r0, r8, ror #5\n", - " 80: fa11 f002 lsls.w r0, r1, r2\n", - " 84: fa31 f002 lsrs.w r0, r1, r2\n", - " 88: fa51 f002 asrs.w r0, r1, r2\n", - " 8c: fa71 f002 rors.w r0, r1, r2\n", - " 90: fa11 f802 lsls.w r8, r1, r2\n", - " 94: fa38 f002 lsrs.w r0, r8, r2\n", - " 98: fa51 f008 asrs.w r0, r1, r8\n", - nullptr -}; -const char* const LoadStoreRegOffsetResults[] = { - " 0: 5888 ldr r0, [r1, r2]\n", - " 2: 5088 str r0, [r1, r2]\n", - " 4: f851 0012 ldr.w r0, [r1, r2, lsl #1]\n", - " 8: f841 0012 str.w r0, [r1, r2, lsl #1]\n", - " c: f851 0032 ldr.w r0, [r1, r2, lsl #3]\n", - " 10: f841 0032 str.w r0, [r1, r2, lsl #3]\n", - " 14: f851 8002 ldr.w r8, [r1, r2]\n", - " 18: f841 8002 str.w r8, [r1, r2]\n", - " 1c: f858 1002 ldr.w r1, [r8, r2]\n", - " 20: f848 2002 str.w r2, [r8, r2]\n", - " 24: f851 0008 ldr.w r0, [r1, r8]\n", - " 28: f841 0008 str.w r0, [r1, r8]\n", - nullptr -}; -const char* const LoadStoreLimitsResults[] = { - " 0: 6fe0 ldr r0, [r4, #124] ; 0x7c\n", - " 2: f8d4 0080 ldr.w r0, [r4, #128] ; 0x80\n", - " 6: 7fe0 ldrb r0, [r4, #31]\n", - " 8: f894 0020 ldrb.w r0, [r4, #32]\n", - " c: 8fe0 ldrh r0, [r4, #62] ; 0x3e\n", - " e: f8b4 0040 ldrh.w r0, [r4, #64] ; 0x40\n", - " 12: f994 001f ldrsb.w r0, [r4, #31]\n", - " 16: f994 0020 ldrsb.w r0, [r4, #32]\n", - " 1a: f9b4 003e ldrsh.w r0, [r4, #62] ; 0x3e\n", - " 1e: f9b4 0040 ldrsh.w r0, [r4, #64] ; 0x40\n", - " 22: 67e0 str r0, [r4, #124] ; 0x7c\n", - " 24: f8c4 0080 str.w r0, [r4, #128] ; 0x80\n", - " 28: 77e0 strb r0, [r4, #31]\n", - " 2a: f884 0020 strb.w r0, [r4, #32]\n", - " 2e: 87e0 strh r0, [r4, #62] ; 0x3e\n", - " 30: f8a4 0040 strh.w r0, [r4, #64] ; 0x40\n", - nullptr -}; -const char* const CompareAndBranchResults[] = { - " 0: b130 cbz r0, 10 <CompareAndBranch+0x10>\n", - " 2: f1bb 0f00 cmp.w fp, #0\n", - " 6: d003 beq.n 10 <CompareAndBranch+0x10>\n", - " 8: b910 cbnz r0, 10 <CompareAndBranch+0x10>\n", - " a: f1bb 0f00 cmp.w fp, #0\n", - " e: d1ff bne.n 10 <CompareAndBranch+0x10>\n", - nullptr -}; - -const char* const AddConstantResults[] = { - " 0: 4608 mov r0, r1\n", - " 2: 1c48 adds r0, r1, #1\n", - " 4: 1dc8 adds r0, r1, #7\n", - " 6: f101 0008 add.w r0, r1, #8\n", - " a: f101 00ff add.w r0, r1, #255 ; 0xff\n", - " e: f501 7080 add.w r0, r1, #256 ; 0x100\n", - " 12: f201 1001 addw r0, r1, #257 ; 0x101\n", - " 16: f601 70ff addw r0, r1, #4095 ; 0xfff\n", - " 1a: f501 5080 add.w r0, r1, #4096 ; 0x1000\n", - " 1e: f46f 5080 mvn.w r0, #4096 ; 0x1000\n", - " 22: 1a08 subs r0, r1, r0\n", - " 24: f241 0002 movw r0, #4098 ; 0x1002\n", - " 28: 1808 adds r0, r1, r0\n", - " 2a: f64f 70ff movw r0, #65535 ; 0xffff\n", - " 2e: 1808 adds r0, r1, r0\n", - " 30: f501 3080 add.w r0, r1, #65536 ; 0x10000\n", - " 34: f101 1001 add.w r0, r1, #65537 ; 0x10001\n", - " 38: f06f 1001 mvn.w r0, #65537 ; 0x10001\n", - " 3c: 1a08 subs r0, r1, r0\n", - " 3e: f240 0003 movw r0, #3\n", - " 42: f2c0 0001 movt r0, #1\n", - " 46: 1808 adds r0, r1, r0\n", - " 48: 1e48 subs r0, r1, #1\n", - " 4a: 1fc8 subs r0, r1, #7\n", - " 4c: f1a1 0008 sub.w r0, r1, #8\n", - " 50: f1a1 00ff sub.w r0, r1, #255 ; 0xff\n", - " 54: f5a1 7080 sub.w r0, r1, #256 ; 0x100\n", - " 58: f2a1 1001 subw r0, r1, #257 ; 0x101\n", - " 5c: f6a1 70ff subw r0, r1, #4095 ; 0xfff\n", - " 60: f5a1 5080 sub.w r0, r1, #4096 ; 0x1000\n", - " 64: f46f 5080 mvn.w r0, #4096 ; 0x1000\n", - " 68: 1808 adds r0, r1, r0\n", - " 6a: f241 0002 movw r0, #4098 ; 0x1002\n", - " 6e: 1a08 subs r0, r1, r0\n", - " 70: f64f 70ff movw r0, #65535 ; 0xffff\n", - " 74: 1a08 subs r0, r1, r0\n", - " 76: f5a1 3080 sub.w r0, r1, #65536 ; 0x10000\n", - " 7a: f1a1 1001 sub.w r0, r1, #65537 ; 0x10001\n", - " 7e: f06f 1001 mvn.w r0, #65537 ; 0x10001\n", - " 82: 1808 adds r0, r1, r0\n", - " 84: f64f 70fd movw r0, #65533 ; 0xfffd\n", - " 88: f6cf 70fe movt r0, #65534 ; 0xfffe\n", - " 8c: 1808 adds r0, r1, r0\n", - " 8e: 3101 adds r1, #1\n", - " 90: 3007 adds r0, #7\n", - " 92: 3108 adds r1, #8\n", - " 94: 30ff adds r0, #255 ; 0xff\n", - " 96: f501 7180 add.w r1, r1, #256 ; 0x100\n", - " 9a: f200 1001 addw r0, r0, #257 ; 0x101\n", - " 9e: f601 71ff addw r1, r1, #4095 ; 0xfff\n", - " a2: f500 5080 add.w r0, r0, #4096 ; 0x1000\n", - " a6: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " aa: eba1 010c sub.w r1, r1, ip\n", - " ae: f241 0c02 movw ip, #4098 ; 0x1002\n", - " b2: 4460 add r0, ip\n", - " b4: f64f 7cff movw ip, #65535 ; 0xffff\n", - " b8: 4461 add r1, ip\n", - " ba: f500 3080 add.w r0, r0, #65536 ; 0x10000\n", - " be: f101 1101 add.w r1, r1, #65537 ; 0x10001\n", - " c2: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " c6: eba0 000c sub.w r0, r0, ip\n", - " ca: f240 0c03 movw ip, #3\n", - " ce: f2c0 0c01 movt ip, #1\n", - " d2: 4461 add r1, ip\n", - " d4: 3801 subs r0, #1\n", - " d6: 3907 subs r1, #7\n", - " d8: 3808 subs r0, #8\n", - " da: 39ff subs r1, #255 ; 0xff\n", - " dc: f5a0 7080 sub.w r0, r0, #256 ; 0x100\n", - " e0: f2a1 1101 subw r1, r1, #257 ; 0x101\n", - " e4: f6a0 70ff subw r0, r0, #4095 ; 0xfff\n", - " e8: f5a1 5180 sub.w r1, r1, #4096 ; 0x1000\n", - " ec: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " f0: 4460 add r0, ip\n", - " f2: f241 0c02 movw ip, #4098 ; 0x1002\n", - " f6: eba1 010c sub.w r1, r1, ip\n", - " fa: f64f 7cff movw ip, #65535 ; 0xffff\n", - " fe: eba0 000c sub.w r0, r0, ip\n", - " 102: f5a1 3180 sub.w r1, r1, #65536 ; 0x10000\n", - " 106: f1a0 1001 sub.w r0, r0, #65537 ; 0x10001\n", - " 10a: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 10e: 4461 add r1, ip\n", - " 110: f64f 7cfd movw ip, #65533 ; 0xfffd\n", - " 114: f6cf 7cfe movt ip, #65534 ; 0xfffe\n", - " 118: 4460 add r0, ip\n", - " 11a: f101 0801 add.w r8, r1, #1\n", - " 11e: f108 0007 add.w r0, r8, #7\n", - " 122: f108 0808 add.w r8, r8, #8\n", - " 126: f101 08ff add.w r8, r1, #255 ; 0xff\n", - " 12a: f508 7080 add.w r0, r8, #256 ; 0x100\n", - " 12e: f208 1801 addw r8, r8, #257 ; 0x101\n", - " 132: f601 78ff addw r8, r1, #4095 ; 0xfff\n", - " 136: f508 5080 add.w r0, r8, #4096 ; 0x1000\n", - " 13a: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 13e: eba8 080c sub.w r8, r8, ip\n", - " 142: f241 0002 movw r0, #4098 ; 0x1002\n", - " 146: 1808 adds r0, r1, r0\n", - " 148: f64f 70ff movw r0, #65535 ; 0xffff\n", - " 14c: eb08 0000 add.w r0, r8, r0\n", - " 150: f508 3880 add.w r8, r8, #65536 ; 0x10000\n", - " 154: f101 1801 add.w r8, r1, #65537 ; 0x10001\n", - " 158: f06f 1001 mvn.w r0, #65537 ; 0x10001\n", - " 15c: eba8 0000 sub.w r0, r8, r0\n", - " 160: f240 0003 movw r0, #3\n", - " 164: f2c0 0001 movt r0, #1\n", - " 168: eb08 0000 add.w r0, r8, r0\n", - " 16c: f108 38ff add.w r8, r8, #4294967295 ; 0xffffffff\n", - " 170: f1a1 0807 sub.w r8, r1, #7\n", - " 174: f1a8 0008 sub.w r0, r8, #8\n", - " 178: f1a8 08ff sub.w r8, r8, #255 ; 0xff\n", - " 17c: f5a1 7880 sub.w r8, r1, #256 ; 0x100\n", - " 180: f2a8 1001 subw r0, r8, #257 ; 0x101\n", - " 184: f6a8 78ff subw r8, r8, #4095 ; 0xfff\n", - " 188: f5a1 5880 sub.w r8, r1, #4096 ; 0x1000\n", - " 18c: f46f 5080 mvn.w r0, #4096 ; 0x1000\n", - " 190: eb08 0000 add.w r0, r8, r0\n", - " 194: f241 0002 movw r0, #4098 ; 0x1002\n", - " 198: 1a08 subs r0, r1, r0\n", - " 19a: f64f 78ff movw r8, #65535 ; 0xffff\n", - " 19e: eba1 0808 sub.w r8, r1, r8\n", - " 1a2: f5a8 3080 sub.w r0, r8, #65536 ; 0x10000\n", - " 1a6: f1a8 1801 sub.w r8, r8, #65537 ; 0x10001\n", - " 1aa: f06f 1801 mvn.w r8, #65537 ; 0x10001\n", - " 1ae: eb01 0808 add.w r8, r1, r8\n", - " 1b2: f64f 70fd movw r0, #65533 ; 0xfffd\n", - " 1b6: f6cf 70fe movt r0, #65534 ; 0xfffe\n", - " 1ba: eb08 0000 add.w r0, r8, r0\n", - " 1be: 4608 mov r0, r1\n", - " 1c0: f101 0001 add.w r0, r1, #1\n", - " 1c4: f101 0007 add.w r0, r1, #7\n", - " 1c8: f101 0008 add.w r0, r1, #8\n", - " 1cc: f101 00ff add.w r0, r1, #255 ; 0xff\n", - " 1d0: f501 7080 add.w r0, r1, #256 ; 0x100\n", - " 1d4: f201 1001 addw r0, r1, #257 ; 0x101\n", - " 1d8: f601 70ff addw r0, r1, #4095 ; 0xfff\n", - " 1dc: f501 5080 add.w r0, r1, #4096 ; 0x1000\n", - " 1e0: f46f 5080 mvn.w r0, #4096 ; 0x1000\n", - " 1e4: eba1 0000 sub.w r0, r1, r0\n", - " 1e8: f241 0002 movw r0, #4098 ; 0x1002\n", - " 1ec: eb01 0000 add.w r0, r1, r0\n", - " 1f0: f64f 70ff movw r0, #65535 ; 0xffff\n", - " 1f4: eb01 0000 add.w r0, r1, r0\n", - " 1f8: f501 3080 add.w r0, r1, #65536 ; 0x10000\n", - " 1fc: f101 1001 add.w r0, r1, #65537 ; 0x10001\n", - " 200: f06f 1001 mvn.w r0, #65537 ; 0x10001\n", - " 204: eba1 0000 sub.w r0, r1, r0\n", - " 208: f240 0003 movw r0, #3\n", - " 20c: f2c0 0001 movt r0, #1\n", - " 210: eb01 0000 add.w r0, r1, r0\n", - " 214: f101 30ff add.w r0, r1, #4294967295 ; 0xffffffff\n", - " 218: f1a1 0007 sub.w r0, r1, #7\n", - " 21c: f1a1 0008 sub.w r0, r1, #8\n", - " 220: f1a1 00ff sub.w r0, r1, #255 ; 0xff\n", - " 224: f5a1 7080 sub.w r0, r1, #256 ; 0x100\n", - " 228: f2a1 1001 subw r0, r1, #257 ; 0x101\n", - " 22c: f6a1 70ff subw r0, r1, #4095 ; 0xfff\n", - " 230: f5a1 5080 sub.w r0, r1, #4096 ; 0x1000\n", - " 234: f46f 5080 mvn.w r0, #4096 ; 0x1000\n", - " 238: eb01 0000 add.w r0, r1, r0\n", - " 23c: f241 0002 movw r0, #4098 ; 0x1002\n", - " 240: eba1 0000 sub.w r0, r1, r0\n", - " 244: f64f 70ff movw r0, #65535 ; 0xffff\n", - " 248: eba1 0000 sub.w r0, r1, r0\n", - " 24c: f5a1 3080 sub.w r0, r1, #65536 ; 0x10000\n", - " 250: f1a1 1001 sub.w r0, r1, #65537 ; 0x10001\n", - " 254: f06f 1001 mvn.w r0, #65537 ; 0x10001\n", - " 258: eb01 0000 add.w r0, r1, r0\n", - " 25c: f64f 70fd movw r0, #65533 ; 0xfffd\n", - " 260: f6cf 70fe movt r0, #65534 ; 0xfffe\n", - " 264: eb01 0000 add.w r0, r1, r0\n", - " 268: f101 0101 add.w r1, r1, #1\n", - " 26c: f100 0007 add.w r0, r0, #7\n", - " 270: f101 0108 add.w r1, r1, #8\n", - " 274: f100 00ff add.w r0, r0, #255 ; 0xff\n", - " 278: f501 7180 add.w r1, r1, #256 ; 0x100\n", - " 27c: f200 1001 addw r0, r0, #257 ; 0x101\n", - " 280: f601 71ff addw r1, r1, #4095 ; 0xfff\n", - " 284: f500 5080 add.w r0, r0, #4096 ; 0x1000\n", - " 288: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 28c: eba1 010c sub.w r1, r1, ip\n", - " 290: f241 0c02 movw ip, #4098 ; 0x1002\n", - " 294: 4460 add r0, ip\n", - " 296: f64f 7cff movw ip, #65535 ; 0xffff\n", - " 29a: 4461 add r1, ip\n", - " 29c: f500 3080 add.w r0, r0, #65536 ; 0x10000\n", - " 2a0: f101 1101 add.w r1, r1, #65537 ; 0x10001\n", - " 2a4: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 2a8: eba0 000c sub.w r0, r0, ip\n", - " 2ac: f240 0c03 movw ip, #3\n", - " 2b0: f2c0 0c01 movt ip, #1\n", - " 2b4: 4461 add r1, ip\n", - " 2b6: f100 30ff add.w r0, r0, #4294967295 ; 0xffffffff\n", - " 2ba: f1a1 0107 sub.w r1, r1, #7\n", - " 2be: f1a0 0008 sub.w r0, r0, #8\n", - " 2c2: f1a1 01ff sub.w r1, r1, #255 ; 0xff\n", - " 2c6: f5a0 7080 sub.w r0, r0, #256 ; 0x100\n", - " 2ca: f2a1 1101 subw r1, r1, #257 ; 0x101\n", - " 2ce: f6a0 70ff subw r0, r0, #4095 ; 0xfff\n", - " 2d2: f5a1 5180 sub.w r1, r1, #4096 ; 0x1000\n", - " 2d6: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 2da: 4460 add r0, ip\n", - " 2dc: f241 0c02 movw ip, #4098 ; 0x1002\n", - " 2e0: eba1 010c sub.w r1, r1, ip\n", - " 2e4: f64f 7cff movw ip, #65535 ; 0xffff\n", - " 2e8: eba0 000c sub.w r0, r0, ip\n", - " 2ec: f5a1 3180 sub.w r1, r1, #65536 ; 0x10000\n", - " 2f0: f1a0 1001 sub.w r0, r0, #65537 ; 0x10001\n", - " 2f4: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 2f8: 4461 add r1, ip\n", - " 2fa: f64f 7cfd movw ip, #65533 ; 0xfffd\n", - " 2fe: f6cf 7cfe movt ip, #65534 ; 0xfffe\n", - " 302: 4460 add r0, ip\n", - " 304: 1c08 adds r0, r1, #0\n", - " 306: 1c48 adds r0, r1, #1\n", - " 308: 1dc8 adds r0, r1, #7\n", - " 30a: f111 0008 adds.w r0, r1, #8\n", - " 30e: f111 00ff adds.w r0, r1, #255 ; 0xff\n", - " 312: f511 7080 adds.w r0, r1, #256 ; 0x100\n", - " 316: f46f 7080 mvn.w r0, #256 ; 0x100\n", - " 31a: 1a08 subs r0, r1, r0\n", - " 31c: f640 70ff movw r0, #4095 ; 0xfff\n", - " 320: 1808 adds r0, r1, r0\n", - " 322: f511 5080 adds.w r0, r1, #4096 ; 0x1000\n", - " 326: f46f 5080 mvn.w r0, #4096 ; 0x1000\n", - " 32a: 1a08 subs r0, r1, r0\n", - " 32c: f241 0002 movw r0, #4098 ; 0x1002\n", - " 330: 1808 adds r0, r1, r0\n", - " 332: f64f 70ff movw r0, #65535 ; 0xffff\n", - " 336: 1808 adds r0, r1, r0\n", - " 338: f511 3080 adds.w r0, r1, #65536 ; 0x10000\n", - " 33c: f111 1001 adds.w r0, r1, #65537 ; 0x10001\n", - " 340: f06f 1001 mvn.w r0, #65537 ; 0x10001\n", - " 344: 1a08 subs r0, r1, r0\n", - " 346: f240 0003 movw r0, #3\n", - " 34a: f2c0 0001 movt r0, #1\n", - " 34e: 1808 adds r0, r1, r0\n", - " 350: 1e48 subs r0, r1, #1\n", - " 352: 1fc8 subs r0, r1, #7\n", - " 354: f1b1 0008 subs.w r0, r1, #8\n", - " 358: f1b1 00ff subs.w r0, r1, #255 ; 0xff\n", - " 35c: f5b1 7080 subs.w r0, r1, #256 ; 0x100\n", - " 360: f46f 7080 mvn.w r0, #256 ; 0x100\n", - " 364: 1808 adds r0, r1, r0\n", - " 366: f640 70ff movw r0, #4095 ; 0xfff\n", - " 36a: 1a08 subs r0, r1, r0\n", - " 36c: f5b1 5080 subs.w r0, r1, #4096 ; 0x1000\n", - " 370: f46f 5080 mvn.w r0, #4096 ; 0x1000\n", - " 374: 1808 adds r0, r1, r0\n", - " 376: f241 0002 movw r0, #4098 ; 0x1002\n", - " 37a: 1a08 subs r0, r1, r0\n", - " 37c: f64f 70ff movw r0, #65535 ; 0xffff\n", - " 380: 1a08 subs r0, r1, r0\n", - " 382: f5b1 3080 subs.w r0, r1, #65536 ; 0x10000\n", - " 386: f1b1 1001 subs.w r0, r1, #65537 ; 0x10001\n", - " 38a: f06f 1001 mvn.w r0, #65537 ; 0x10001\n", - " 38e: 1808 adds r0, r1, r0\n", - " 390: f64f 70fd movw r0, #65533 ; 0xfffd\n", - " 394: f6cf 70fe movt r0, #65534 ; 0xfffe\n", - " 398: 1808 adds r0, r1, r0\n", - " 39a: 3000 adds r0, #0\n", - " 39c: 3101 adds r1, #1\n", - " 39e: 3007 adds r0, #7\n", - " 3a0: 3108 adds r1, #8\n", - " 3a2: 30ff adds r0, #255 ; 0xff\n", - " 3a4: f511 7180 adds.w r1, r1, #256 ; 0x100\n", - " 3a8: f46f 7c80 mvn.w ip, #256 ; 0x100\n", - " 3ac: ebb0 000c subs.w r0, r0, ip\n", - " 3b0: f640 7cff movw ip, #4095 ; 0xfff\n", - " 3b4: eb11 010c adds.w r1, r1, ip\n", - " 3b8: f510 5080 adds.w r0, r0, #4096 ; 0x1000\n", - " 3bc: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 3c0: ebb1 010c subs.w r1, r1, ip\n", - " 3c4: f241 0c02 movw ip, #4098 ; 0x1002\n", - " 3c8: eb10 000c adds.w r0, r0, ip\n", - " 3cc: f64f 7cff movw ip, #65535 ; 0xffff\n", - " 3d0: eb11 010c adds.w r1, r1, ip\n", - " 3d4: f510 3080 adds.w r0, r0, #65536 ; 0x10000\n", - " 3d8: f111 1101 adds.w r1, r1, #65537 ; 0x10001\n", - " 3dc: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 3e0: ebb0 000c subs.w r0, r0, ip\n", - " 3e4: f240 0c03 movw ip, #3\n", - " 3e8: f2c0 0c01 movt ip, #1\n", - " 3ec: eb11 010c adds.w r1, r1, ip\n", - " 3f0: 3801 subs r0, #1\n", - " 3f2: 3907 subs r1, #7\n", - " 3f4: 3808 subs r0, #8\n", - " 3f6: 39ff subs r1, #255 ; 0xff\n", - " 3f8: f5b0 7080 subs.w r0, r0, #256 ; 0x100\n", - " 3fc: f46f 7c80 mvn.w ip, #256 ; 0x100\n", - " 400: eb11 010c adds.w r1, r1, ip\n", - " 404: f640 7cff movw ip, #4095 ; 0xfff\n", - " 408: ebb0 000c subs.w r0, r0, ip\n", - " 40c: f5b1 5180 subs.w r1, r1, #4096 ; 0x1000\n", - " 410: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 414: eb10 000c adds.w r0, r0, ip\n", - " 418: f241 0c02 movw ip, #4098 ; 0x1002\n", - " 41c: ebb1 010c subs.w r1, r1, ip\n", - " 420: f64f 7cff movw ip, #65535 ; 0xffff\n", - " 424: ebb0 000c subs.w r0, r0, ip\n", - " 428: f5b1 3180 subs.w r1, r1, #65536 ; 0x10000\n", - " 42c: f1b0 1001 subs.w r0, r0, #65537 ; 0x10001\n", - " 430: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 434: eb11 010c adds.w r1, r1, ip\n", - " 438: f64f 7cfd movw ip, #65533 ; 0xfffd\n", - " 43c: f6cf 7cfe movt ip, #65534 ; 0xfffe\n", - " 440: eb10 000c adds.w r0, r0, ip\n", - " 444: bf08 it eq\n", - " 446: f111 0001 addseq.w r0, r1, #1\n", - " 44a: bf18 it ne\n", - " 44c: 1c48 addne r0, r1, #1\n", - " 44e: bfa8 it ge\n", - " 450: f110 0001 addsge.w r0, r0, #1\n", - " 454: bfd8 it le\n", - " 456: 3001 addle r0, #1\n", - nullptr -}; - -const char* const CmpConstantResults[] = { - " 0: 2800 cmp r0, #0\n", - " 2: 2901 cmp r1, #1\n", - " 4: 2807 cmp r0, #7\n", - " 6: 2908 cmp r1, #8\n", - " 8: 28ff cmp r0, #255 ; 0xff\n", - " a: f5b1 7f80 cmp.w r1, #256 ; 0x100\n", - " e: f46f 7c80 mvn.w ip, #256 ; 0x100\n", - " 12: eb10 0f0c cmn.w r0, ip\n", - " 16: f640 7cff movw ip, #4095 ; 0xfff\n", - " 1a: 4561 cmp r1, ip\n", - " 1c: f5b0 5f80 cmp.w r0, #4096 ; 0x1000\n", - " 20: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 24: eb11 0f0c cmn.w r1, ip\n", - " 28: f241 0c02 movw ip, #4098 ; 0x1002\n", - " 2c: 4560 cmp r0, ip\n", - " 2e: f64f 7cff movw ip, #65535 ; 0xffff\n", - " 32: 4561 cmp r1, ip\n", - " 34: f5b0 3f80 cmp.w r0, #65536 ; 0x10000\n", - " 38: f1b1 1f01 cmp.w r1, #65537 ; 0x10001\n", - " 3c: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 40: eb10 0f0c cmn.w r0, ip\n", - " 44: f240 0c03 movw ip, #3\n", - " 48: f2c0 0c01 movt ip, #1\n", - " 4c: 4561 cmp r1, ip\n", - " 4e: f1b0 3fff cmp.w r0, #4294967295 ; 0xffffffff\n", - " 52: f111 0f07 cmn.w r1, #7\n", - " 56: f110 0f08 cmn.w r0, #8\n", - " 5a: f111 0fff cmn.w r1, #255 ; 0xff\n", - " 5e: f510 7f80 cmn.w r0, #256 ; 0x100\n", - " 62: f46f 7c80 mvn.w ip, #256 ; 0x100\n", - " 66: 4561 cmp r1, ip\n", - " 68: f640 7cff movw ip, #4095 ; 0xfff\n", - " 6c: eb10 0f0c cmn.w r0, ip\n", - " 70: f511 5f80 cmn.w r1, #4096 ; 0x1000\n", - " 74: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 78: 4560 cmp r0, ip\n", - " 7a: f241 0c02 movw ip, #4098 ; 0x1002\n", - " 7e: eb11 0f0c cmn.w r1, ip\n", - " 82: f64f 7cff movw ip, #65535 ; 0xffff\n", - " 86: eb10 0f0c cmn.w r0, ip\n", - " 8a: f511 3f80 cmn.w r1, #65536 ; 0x10000\n", - " 8e: f110 1f01 cmn.w r0, #65537 ; 0x10001\n", - " 92: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 96: 4561 cmp r1, ip\n", - " 98: f64f 7cfd movw ip, #65533 ; 0xfffd\n", - " 9c: f6cf 7cfe movt ip, #65534 ; 0xfffe\n", - " a0: 4560 cmp r0, ip\n", - " a2: f1b8 0f00 cmp.w r8, #0\n", - " a6: f1b9 0f01 cmp.w r9, #1\n", - " aa: f1b8 0f07 cmp.w r8, #7\n", - " ae: f1b9 0f08 cmp.w r9, #8\n", - " b2: f1b8 0fff cmp.w r8, #255 ; 0xff\n", - " b6: f5b9 7f80 cmp.w r9, #256 ; 0x100\n", - " ba: f46f 7c80 mvn.w ip, #256 ; 0x100\n", - " be: eb18 0f0c cmn.w r8, ip\n", - " c2: f640 7cff movw ip, #4095 ; 0xfff\n", - " c6: 45e1 cmp r9, ip\n", - " c8: f5b8 5f80 cmp.w r8, #4096 ; 0x1000\n", - " cc: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " d0: eb19 0f0c cmn.w r9, ip\n", - " d4: f241 0c02 movw ip, #4098 ; 0x1002\n", - " d8: 45e0 cmp r8, ip\n", - " da: f64f 7cff movw ip, #65535 ; 0xffff\n", - " de: 45e1 cmp r9, ip\n", - " e0: f5b8 3f80 cmp.w r8, #65536 ; 0x10000\n", - " e4: f1b9 1f01 cmp.w r9, #65537 ; 0x10001\n", - " e8: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " ec: eb18 0f0c cmn.w r8, ip\n", - " f0: f240 0c03 movw ip, #3\n", - " f4: f2c0 0c01 movt ip, #1\n", - " f8: 45e1 cmp r9, ip\n", - " fa: f1b8 3fff cmp.w r8, #4294967295 ; 0xffffffff\n", - " fe: f119 0f07 cmn.w r9, #7\n", - " 102: f118 0f08 cmn.w r8, #8\n", - " 106: f119 0fff cmn.w r9, #255 ; 0xff\n", - " 10a: f518 7f80 cmn.w r8, #256 ; 0x100\n", - " 10e: f46f 7c80 mvn.w ip, #256 ; 0x100\n", - " 112: 45e1 cmp r9, ip\n", - " 114: f640 7cff movw ip, #4095 ; 0xfff\n", - " 118: eb18 0f0c cmn.w r8, ip\n", - " 11c: f519 5f80 cmn.w r9, #4096 ; 0x1000\n", - " 120: f46f 5c80 mvn.w ip, #4096 ; 0x1000\n", - " 124: 45e0 cmp r8, ip\n", - " 126: f241 0c02 movw ip, #4098 ; 0x1002\n", - " 12a: eb19 0f0c cmn.w r9, ip\n", - " 12e: f64f 7cff movw ip, #65535 ; 0xffff\n", - " 132: eb18 0f0c cmn.w r8, ip\n", - " 136: f519 3f80 cmn.w r9, #65536 ; 0x10000\n", - " 13a: f118 1f01 cmn.w r8, #65537 ; 0x10001\n", - " 13e: f06f 1c01 mvn.w ip, #65537 ; 0x10001\n", - " 142: 45e1 cmp r9, ip\n", - " 144: f64f 7cfd movw ip, #65533 ; 0xfffd\n", - " 148: f6cf 7cfe movt ip, #65534 ; 0xfffe\n", - " 14c: 45e0 cmp r8, ip\n", - nullptr -}; - const char* const VixlJniHelpersResults[] = { " 0: e92d 4de0 stmdb sp!, {r5, r6, r7, r8, sl, fp, lr}\n", " 4: ed2d 8a10 vpush {s16-s31}\n", @@ -5595,7 +136,7 @@ const char* const VixlJniHelpersResults[] = { " 1dc: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1e0: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1e4: f000 b802 b.w 1ec <VixlJniHelpers+0x1ec>\n", - " 1e8: f000 b818 b.w 21c <VixlJniHelpers+0x21c>\n", + " 1e8: f000 b81b b.w 222 <VixlJniHelpers+0x222>\n", " 1ec: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1f0: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1f4: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", @@ -5608,10 +149,12 @@ const char* const VixlJniHelpersResults[] = { " 210: b008 add sp, #32\n", " 212: b009 add sp, #36 ; 0x24\n", " 214: ecbd 8a10 vpop {s16-s31}\n", - " 218: e8bd 8de0 ldmia.w sp!, {r5, r6, r7, r8, sl, fp, pc}\n", - " 21c: 4660 mov r0, ip\n", - " 21e: f8d9 c2c0 ldr.w ip, [r9, #704] ; 0x2c0\n", - " 222: 47e0 blx ip\n", + " 218: e8bd 4de0 ldmia.w sp!, {r5, r6, r7, r8, sl, fp, lr}\n", + " 21c: f8d9 8034 ldr.w r8, [r9, #52] ; 0x34\n", + " 220: 4770 bx lr\n", + " 222: 4660 mov r0, ip\n", + " 224: f8d9 c2c0 ldr.w ip, [r9, #704] ; 0x2c0\n", + " 228: 47e0 blx ip\n", nullptr }; @@ -5718,55 +261,6 @@ const char* const VixlStoreToOffsetResults[] = { std::map<std::string, const char* const*> test_results; void setup_results() { - test_results["SimpleMov"] = SimpleMovResults; - test_results["SimpleMov32"] = SimpleMov32Results; - test_results["SimpleMovAdd"] = SimpleMovAddResults; - test_results["DataProcessingRegister"] = DataProcessingRegisterResults; - test_results["DataProcessingImmediate"] = DataProcessingImmediateResults; - test_results["DataProcessingModifiedImmediate"] = DataProcessingModifiedImmediateResults; - test_results["DataProcessingModifiedImmediates"] = DataProcessingModifiedImmediatesResults; - test_results["DataProcessingShiftedRegister"] = DataProcessingShiftedRegisterResults; - test_results["ShiftImmediate"] = ShiftImmediateResults; - test_results["BasicLoad"] = BasicLoadResults; - test_results["BasicStore"] = BasicStoreResults; - test_results["ComplexLoad"] = ComplexLoadResults; - test_results["ComplexStore"] = ComplexStoreResults; - test_results["NegativeLoadStore"] = NegativeLoadStoreResults; - test_results["SimpleLoadStoreDual"] = SimpleLoadStoreDualResults; - test_results["ComplexLoadStoreDual"] = ComplexLoadStoreDualResults; - test_results["NegativeLoadStoreDual"] = NegativeLoadStoreDualResults; - test_results["SimpleBranch"] = SimpleBranchResults; - test_results["LongBranch"] = LongBranchResults; - test_results["LoadMultiple"] = LoadMultipleResults; - test_results["StoreMultiple"] = StoreMultipleResults; - test_results["MovWMovT"] = MovWMovTResults; - test_results["SpecialAddSub"] = SpecialAddSubResults; - test_results["LoadFromOffset"] = LoadFromOffsetResults; - test_results["StoreToOffset"] = StoreToOffsetResults; - test_results["IfThen"] = IfThenResults; - test_results["CbzCbnz"] = CbzCbnzResults; - test_results["Multiply"] = MultiplyResults; - test_results["Divide"] = DivideResults; - test_results["VMov"] = VMovResults; - test_results["BasicFloatingPoint"] = BasicFloatingPointResults; - test_results["FloatingPointConversions"] = FloatingPointConversionsResults; - test_results["FloatingPointComparisons"] = FloatingPointComparisonsResults; - test_results["Calls"] = CallsResults; - test_results["Breakpoint"] = BreakpointResults; - test_results["StrR1"] = StrR1Results; - test_results["VPushPop"] = VPushPopResults; - test_results["Max16BitBranch"] = Max16BitBranchResults; - test_results["Branch32"] = Branch32Results; - test_results["CompareAndBranchMax"] = CompareAndBranchMaxResults; - test_results["CompareAndBranchRelocation16"] = CompareAndBranchRelocation16Results; - test_results["CompareAndBranchRelocation32"] = CompareAndBranchRelocation32Results; - test_results["MixedBranch32"] = MixedBranch32Results; - test_results["Shifts"] = ShiftsResults; - test_results["LoadStoreRegOffset"] = LoadStoreRegOffsetResults; - test_results["LoadStoreLimits"] = LoadStoreLimitsResults; - test_results["CompareAndBranch"] = CompareAndBranchResults; - test_results["AddConstant"] = AddConstantResults; - test_results["CmpConstant"] = CmpConstantResults; test_results["VixlJniHelpers"] = VixlJniHelpersResults; test_results["VixlStoreToOffset"] = VixlStoreToOffsetResults; test_results["VixlLoadFromOffset"] = VixlLoadFromOffsetResults; diff --git a/compiler/utils/atomic_method_ref_map-inl.h b/compiler/utils/atomic_dex_ref_map-inl.h index ad3a099eb6..c41d8fc071 100644 --- a/compiler/utils/atomic_method_ref_map-inl.h +++ b/compiler/utils/atomic_dex_ref_map-inl.h @@ -14,72 +14,72 @@ * limitations under the License. */ -#ifndef ART_COMPILER_UTILS_ATOMIC_METHOD_REF_MAP_INL_H_ -#define ART_COMPILER_UTILS_ATOMIC_METHOD_REF_MAP_INL_H_ +#ifndef ART_COMPILER_UTILS_ATOMIC_DEX_REF_MAP_INL_H_ +#define ART_COMPILER_UTILS_ATOMIC_DEX_REF_MAP_INL_H_ -#include "atomic_method_ref_map.h" +#include "atomic_dex_ref_map.h" #include "dex_file-inl.h" namespace art { template <typename T> -inline typename AtomicMethodRefMap<T>::InsertResult AtomicMethodRefMap<T>::Insert( - MethodReference ref, +inline typename AtomicDexRefMap<T>::InsertResult AtomicDexRefMap<T>::Insert( + DexFileReference ref, const T& expected, const T& desired) { ElementArray* const array = GetArray(ref.dex_file); if (array == nullptr) { return kInsertResultInvalidDexFile; } - return (*array)[ref.dex_method_index].CompareExchangeStrongSequentiallyConsistent( - expected, desired) + DCHECK_LT(ref.index, array->size()); + return (*array)[ref.index].CompareExchangeStrongSequentiallyConsistent(expected, desired) ? kInsertResultSuccess : kInsertResultCASFailure; } template <typename T> -inline bool AtomicMethodRefMap<T>::Get(MethodReference ref, T* out) const { +inline bool AtomicDexRefMap<T>::Get(DexFileReference ref, T* out) const { const ElementArray* const array = GetArray(ref.dex_file); if (array == nullptr) { return false; } - *out = (*array)[ref.dex_method_index].LoadRelaxed(); + *out = (*array)[ref.index].LoadRelaxed(); return true; } template <typename T> -inline void AtomicMethodRefMap<T>::AddDexFile(const DexFile* dex_file) { - arrays_.Put(dex_file, std::move(ElementArray(dex_file->NumMethodIds()))); +inline void AtomicDexRefMap<T>::AddDexFile(const DexFile* dex_file, size_t max_index) { + arrays_.Put(dex_file, std::move(ElementArray(max_index))); } template <typename T> -inline typename AtomicMethodRefMap<T>::ElementArray* AtomicMethodRefMap<T>::GetArray( +inline typename AtomicDexRefMap<T>::ElementArray* AtomicDexRefMap<T>::GetArray( const DexFile* dex_file) { auto it = arrays_.find(dex_file); return (it != arrays_.end()) ? &it->second : nullptr; } template <typename T> -inline const typename AtomicMethodRefMap<T>::ElementArray* AtomicMethodRefMap<T>::GetArray( +inline const typename AtomicDexRefMap<T>::ElementArray* AtomicDexRefMap<T>::GetArray( const DexFile* dex_file) const { auto it = arrays_.find(dex_file); return (it != arrays_.end()) ? &it->second : nullptr; } template <typename T> template <typename Visitor> -inline void AtomicMethodRefMap<T>::Visit(const Visitor& visitor) { +inline void AtomicDexRefMap<T>::Visit(const Visitor& visitor) { for (auto& pair : arrays_) { const DexFile* dex_file = pair.first; const ElementArray& elements = pair.second; for (size_t i = 0; i < elements.size(); ++i) { - visitor(MethodReference(dex_file, i), elements[i].LoadRelaxed()); + visitor(DexFileReference(dex_file, i), elements[i].LoadRelaxed()); } } } template <typename T> -inline void AtomicMethodRefMap<T>::ClearEntries() { +inline void AtomicDexRefMap<T>::ClearEntries() { for (auto& it : arrays_) { for (auto& element : it.second) { element.StoreRelaxed(nullptr); @@ -89,4 +89,4 @@ inline void AtomicMethodRefMap<T>::ClearEntries() { } // namespace art -#endif // ART_COMPILER_UTILS_ATOMIC_METHOD_REF_MAP_INL_H_ +#endif // ART_COMPILER_UTILS_ATOMIC_DEX_REF_MAP_INL_H_ diff --git a/compiler/utils/atomic_method_ref_map.h b/compiler/utils/atomic_dex_ref_map.h index fed848f563..2da4ffa27b 100644 --- a/compiler/utils/atomic_method_ref_map.h +++ b/compiler/utils/atomic_dex_ref_map.h @@ -14,11 +14,11 @@ * limitations under the License. */ -#ifndef ART_COMPILER_UTILS_ATOMIC_METHOD_REF_MAP_H_ -#define ART_COMPILER_UTILS_ATOMIC_METHOD_REF_MAP_H_ +#ifndef ART_COMPILER_UTILS_ATOMIC_DEX_REF_MAP_H_ +#define ART_COMPILER_UTILS_ATOMIC_DEX_REF_MAP_H_ #include "base/dchecked_vector.h" -#include "method_reference.h" +#include "dex_file.h" #include "safe_map.h" namespace art { @@ -27,10 +27,10 @@ class DexFile; // Used by CompilerCallbacks to track verification information from the Runtime. template <typename T> -class AtomicMethodRefMap { +class AtomicDexRefMap { public: - explicit AtomicMethodRefMap() {} - ~AtomicMethodRefMap() {} + explicit AtomicDexRefMap() {} + ~AtomicDexRefMap() {} // Atomically swap the element in if the existing value matches expected. enum InsertResult { @@ -38,14 +38,14 @@ class AtomicMethodRefMap { kInsertResultCASFailure, kInsertResultSuccess, }; - InsertResult Insert(MethodReference ref, const T& expected, const T& desired); + InsertResult Insert(DexFileReference ref, const T& expected, const T& desired); // Retreive an item, returns false if the dex file is not added. - bool Get(MethodReference ref, T* out) const; + bool Get(DexFileReference ref, T* out) const; // Dex files must be added before method references belonging to them can be used as keys. Not // thread safe. - void AddDexFile(const DexFile* dex_file); + void AddDexFile(const DexFile* dex_file, size_t max_index); bool HaveDexFile(const DexFile* dex_file) const { return arrays_.find(dex_file) != arrays_.end(); @@ -70,4 +70,4 @@ class AtomicMethodRefMap { } // namespace art -#endif // ART_COMPILER_UTILS_ATOMIC_METHOD_REF_MAP_H_ +#endif // ART_COMPILER_UTILS_ATOMIC_DEX_REF_MAP_H_ diff --git a/compiler/utils/atomic_method_ref_map_test.cc b/compiler/utils/atomic_dex_ref_map_test.cc index 9e5bf4bbe1..ae19a9c6da 100644 --- a/compiler/utils/atomic_method_ref_map_test.cc +++ b/compiler/utils/atomic_dex_ref_map_test.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "atomic_method_ref_map-inl.h" +#include "atomic_dex_ref_map-inl.h" #include <memory> @@ -25,46 +25,46 @@ namespace art { -class AtomicMethodRefMapTest : public CommonRuntimeTest {}; +class AtomicDexRefMapTest : public CommonRuntimeTest {}; -TEST_F(AtomicMethodRefMapTest, RunTests) { +TEST_F(AtomicDexRefMapTest, RunTests) { ScopedObjectAccess soa(Thread::Current()); std::unique_ptr<const DexFile> dex(OpenTestDexFile("Interfaces")); ASSERT_TRUE(dex != nullptr); - using Map = AtomicMethodRefMap<int>; + using Map = AtomicDexRefMap<int>; Map map; int value = 123; // Error case: Not already inserted. - EXPECT_FALSE(map.Get(MethodReference(dex.get(), 1), &value)); + EXPECT_FALSE(map.Get(DexFileReference(dex.get(), 1), &value)); EXPECT_FALSE(map.HaveDexFile(dex.get())); // Error case: Dex file not registered. - EXPECT_TRUE(map.Insert(MethodReference(dex.get(), 1), 0, 1) == Map::kInsertResultInvalidDexFile); - map.AddDexFile(dex.get()); + EXPECT_TRUE(map.Insert(DexFileReference(dex.get(), 1), 0, 1) == Map::kInsertResultInvalidDexFile); + map.AddDexFile(dex.get(), dex->NumMethodIds()); EXPECT_TRUE(map.HaveDexFile(dex.get())); EXPECT_GT(dex->NumMethodIds(), 10u); // After we have added the get should succeed but return the default value. - EXPECT_TRUE(map.Get(MethodReference(dex.get(), 1), &value)); + EXPECT_TRUE(map.Get(DexFileReference(dex.get(), 1), &value)); EXPECT_EQ(value, 0); // Actually insert an item and make sure we can retreive it. static const int kInsertValue = 44; - EXPECT_TRUE(map.Insert(MethodReference(dex.get(), 1), 0, kInsertValue) == + EXPECT_TRUE(map.Insert(DexFileReference(dex.get(), 1), 0, kInsertValue) == Map::kInsertResultSuccess); - EXPECT_TRUE(map.Get(MethodReference(dex.get(), 1), &value)); + EXPECT_TRUE(map.Get(DexFileReference(dex.get(), 1), &value)); EXPECT_EQ(value, kInsertValue); static const int kInsertValue2 = 123; - EXPECT_TRUE(map.Insert(MethodReference(dex.get(), 2), 0, kInsertValue2) == + EXPECT_TRUE(map.Insert(DexFileReference(dex.get(), 2), 0, kInsertValue2) == Map::kInsertResultSuccess); - EXPECT_TRUE(map.Get(MethodReference(dex.get(), 1), &value)); + EXPECT_TRUE(map.Get(DexFileReference(dex.get(), 1), &value)); EXPECT_EQ(value, kInsertValue); - EXPECT_TRUE(map.Get(MethodReference(dex.get(), 2), &value)); + EXPECT_TRUE(map.Get(DexFileReference(dex.get(), 2), &value)); EXPECT_EQ(value, kInsertValue2); // Error case: Incorrect expected value for CAS. - EXPECT_TRUE(map.Insert(MethodReference(dex.get(), 1), 0, kInsertValue + 1) == + EXPECT_TRUE(map.Insert(DexFileReference(dex.get(), 1), 0, kInsertValue + 1) == Map::kInsertResultCASFailure); // Correctly overwrite the value and verify. - EXPECT_TRUE(map.Insert(MethodReference(dex.get(), 1), kInsertValue, kInsertValue + 1) == + EXPECT_TRUE(map.Insert(DexFileReference(dex.get(), 1), kInsertValue, kInsertValue + 1) == Map::kInsertResultSuccess); - EXPECT_TRUE(map.Get(MethodReference(dex.get(), 1), &value)); + EXPECT_TRUE(map.Get(DexFileReference(dex.get(), 1), &value)); EXPECT_EQ(value, kInsertValue + 1); } diff --git a/compiler/utils/intrusive_forward_list.h b/compiler/utils/intrusive_forward_list.h index b5fc2f2456..5a358ac2c4 100644 --- a/compiler/utils/intrusive_forward_list.h +++ b/compiler/utils/intrusive_forward_list.h @@ -23,6 +23,7 @@ #include <memory> #include <type_traits> +#include "base/casts.h" #include "base/logging.h" #include "base/macros.h" @@ -42,10 +43,19 @@ struct IntrusiveForwardListHook { mutable const IntrusiveForwardListHook* next_hook; }; +template <typename Derived, typename Tag = void> +struct IntrusiveForwardListNode : public IntrusiveForwardListHook { +}; + template <typename T, IntrusiveForwardListHook T::* NextPtr = &T::hook> -class IntrusiveForwardListMemberHook; +class IntrusiveForwardListMemberHookTraits; + +template <typename T, typename Tag = void> +class IntrusiveForwardListBaseHookTraits; -template <typename T, typename HookTraits = IntrusiveForwardListMemberHook<T>> +template <typename T, + typename HookTraits = + IntrusiveForwardListBaseHookTraits<typename std::remove_const<T>::type>> class IntrusiveForwardList; template <typename T, typename HookTraits> @@ -435,7 +445,7 @@ bool operator>=(const IntrusiveForwardList<T, HookTraits>& lhs, } template <typename T, IntrusiveForwardListHook T::* NextPtr> -class IntrusiveForwardListMemberHook { +class IntrusiveForwardListMemberHookTraits { public: static const IntrusiveForwardListHook* GetHook(const T* value) { return &(value->*NextPtr); @@ -447,6 +457,20 @@ class IntrusiveForwardListMemberHook { } }; +template <typename T, typename Tag> +class IntrusiveForwardListBaseHookTraits { + public: + static const IntrusiveForwardListHook* GetHook(const T* value) { + // Explicit conversion to the "node" followed by implicit conversion to the "hook". + return static_cast<const IntrusiveForwardListNode<T, Tag>*>(value); + } + + static T* GetValue(const IntrusiveForwardListHook* hook) { + return down_cast<T*>(down_cast<IntrusiveForwardListNode<T, Tag>*>( + const_cast<IntrusiveForwardListHook*>(hook))); + } +}; + } // namespace art #endif // ART_COMPILER_UTILS_INTRUSIVE_FORWARD_LIST_H_ diff --git a/compiler/utils/intrusive_forward_list_test.cc b/compiler/utils/intrusive_forward_list_test.cc index f2efa4dd15..939676cdc8 100644 --- a/compiler/utils/intrusive_forward_list_test.cc +++ b/compiler/utils/intrusive_forward_list_test.cc @@ -23,13 +23,14 @@ namespace art { -struct IFLTestValue { +struct IFLTestValue : public IntrusiveForwardListNode<IFLTestValue> { // Deliberately not explicit. - IFLTestValue(int v) : hook(), value(v) { } // NOLINT(runtime/explicit) + IFLTestValue(int v) : value(v) { } // NOLINT(runtime/explicit) - IntrusiveForwardListHook hook; int value; }; +using IFLTestValueList = IntrusiveForwardList<IFLTestValue>; +using ConstIFLTestValueList = IntrusiveForwardList<const IFLTestValue>; bool operator==(const IFLTestValue& lhs, const IFLTestValue& rhs) { return lhs.value == rhs.value; @@ -39,6 +40,24 @@ bool operator<(const IFLTestValue& lhs, const IFLTestValue& rhs) { return lhs.value < rhs.value; } +struct IFLTestValue2 { + // Deliberately not explicit. + IFLTestValue2(int v) : hook(), value(v) { } // NOLINT(runtime/explicit) + + IntrusiveForwardListHook hook; + int value; +}; +using IFLTestValue2List = + IntrusiveForwardList<IFLTestValue2, IntrusiveForwardListMemberHookTraits<IFLTestValue2>>; + +bool operator==(const IFLTestValue2& lhs, const IFLTestValue2& rhs) { + return lhs.value == rhs.value; +} + +bool operator<(const IFLTestValue2& lhs, const IFLTestValue2& rhs) { + return lhs.value < rhs.value; +} + #define ASSERT_LISTS_EQUAL(expected, value) \ do { \ ASSERT_EQ((expected).empty(), (value).empty()); \ @@ -47,16 +66,82 @@ bool operator<(const IFLTestValue& lhs, const IFLTestValue& rhs) { ASSERT_TRUE(std::equal((expected).begin(), (expected).end(), (value).begin())); \ } while (false) -TEST(IntrusiveForwardList, IteratorToConstIterator) { - IntrusiveForwardList<IFLTestValue> ifl; - IntrusiveForwardList<IFLTestValue>::iterator begin = ifl.begin(); - IntrusiveForwardList<IFLTestValue>::const_iterator cbegin = ifl.cbegin(); - IntrusiveForwardList<IFLTestValue>::const_iterator converted_begin = begin; +class IntrusiveForwardListTest : public testing::Test { + public: + template <typename ListType> + void IteratorToConstIterator(); + + template <typename ListType> + void IteratorOperators(); + + template <typename ListType> + void ConstructRange(); + + template <typename ListType> + void Assign(); + + template <typename ListType> + void PushPop(); + + template <typename ListType> + void InsertAfter1(); + + template <typename ListType> + void InsertAfter2(); + + template <typename ListType> + void EraseAfter1(); + + template <typename ListType> + void EraseAfter2(); + + template <typename ListType> + void SwapClear(); + + template <typename ListType> + void SpliceAfter(); + + template <typename ListType> + void Remove(); + + template <typename ListType> + void Unique(); + + template <typename ListType> + void Merge(); + + template <typename ListType> + void Sort1(); + + template <typename ListType> + void Sort2(); + + template <typename ListType> + void Reverse(); + + template <typename ListType> + void ModifyValue(); +}; + +template <typename ListType> +void IntrusiveForwardListTest::IteratorToConstIterator() { + ListType ifl; + typename ListType::iterator begin = ifl.begin(); + typename ListType::const_iterator cbegin = ifl.cbegin(); + typename ListType::const_iterator converted_begin = begin; ASSERT_TRUE(converted_begin == cbegin); } -TEST(IntrusiveForwardList, IteratorOperators) { - IntrusiveForwardList<IFLTestValue> ifl; +TEST_F(IntrusiveForwardListTest, IteratorToConstIterator) { + IteratorToConstIterator<IFLTestValueList>(); + IteratorToConstIterator<ConstIFLTestValueList>(); + IteratorToConstIterator<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::IteratorOperators() { + using ValueType = typename ListType::value_type; + ListType ifl; ASSERT_TRUE(ifl.begin() == ifl.cbegin()); ASSERT_FALSE(ifl.begin() != ifl.cbegin()); ASSERT_TRUE(ifl.end() == ifl.cend()); @@ -65,37 +150,61 @@ TEST(IntrusiveForwardList, IteratorOperators) { ASSERT_TRUE(ifl.begin() == ifl.end()); // Empty. ASSERT_FALSE(ifl.begin() != ifl.end()); // Empty. - IFLTestValue value(1); + ValueType value(1); ifl.insert_after(ifl.cbefore_begin(), value); ASSERT_FALSE(ifl.begin() == ifl.end()); // Not empty. ASSERT_TRUE(ifl.begin() != ifl.end()); // Not empty. } -TEST(IntrusiveForwardList, ConstructRange) { +TEST_F(IntrusiveForwardListTest, IteratorOperators) { + IteratorOperators<IFLTestValueList>(); + IteratorOperators<ConstIFLTestValueList>(); + IteratorOperators<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::ConstructRange() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 1, 2, 7 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); } -TEST(IntrusiveForwardList, Assign) { +TEST_F(IntrusiveForwardListTest, ConstructRange) { + ConstructRange<IFLTestValueList>(); + ConstructRange<ConstIFLTestValueList>(); + ConstructRange<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::Assign() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref1({ 2, 8, 5 }); - std::vector<IFLTestValue> storage1(ref1.begin(), ref1.end()); - IntrusiveForwardList<IFLTestValue> ifl; + std::vector<ValueType> storage1(ref1.begin(), ref1.end()); + ListType ifl; ifl.assign(storage1.begin(), storage1.end()); ASSERT_LISTS_EQUAL(ref1, ifl); std::forward_list<int> ref2({ 7, 1, 3 }); - std::vector<IFLTestValue> storage2(ref2.begin(), ref2.end()); + std::vector<ValueType> storage2(ref2.begin(), ref2.end()); ifl.assign(storage2.begin(), storage2.end()); ASSERT_LISTS_EQUAL(ref2, ifl); } -TEST(IntrusiveForwardList, PushPop) { - IFLTestValue value3(3); - IFLTestValue value7(7); +TEST_F(IntrusiveForwardListTest, Assign) { + Assign<IFLTestValueList>(); + Assign<ConstIFLTestValueList>(); + Assign<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::PushPop() { + using ValueType = typename ListType::value_type; + ValueType value3(3); + ValueType value7(7); std::forward_list<int> ref; - IntrusiveForwardList<IFLTestValue> ifl; + ListType ifl; ASSERT_LISTS_EQUAL(ref, ifl); ref.push_front(3); ifl.push_front(value3); @@ -114,13 +223,21 @@ TEST(IntrusiveForwardList, PushPop) { ASSERT_LISTS_EQUAL(ref, ifl); } -TEST(IntrusiveForwardList, InsertAfter1) { - IFLTestValue value4(4); - IFLTestValue value8(8); - IFLTestValue value5(5); - IFLTestValue value3(3); +TEST_F(IntrusiveForwardListTest, PushPop) { + PushPop<IFLTestValueList>(); + PushPop<ConstIFLTestValueList>(); + PushPop<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::InsertAfter1() { + using ValueType = typename ListType::value_type; + ValueType value4(4); + ValueType value8(8); + ValueType value5(5); + ValueType value3(3); std::forward_list<int> ref; - IntrusiveForwardList<IFLTestValue> ifl; + ListType ifl; auto ref_it = ref.insert_after(ref.before_begin(), 4); auto ifl_it = ifl.insert_after(ifl.before_begin(), value4); @@ -149,23 +266,31 @@ TEST(IntrusiveForwardList, InsertAfter1) { ASSERT_EQ(*ref_it, *ifl_it); } -TEST(IntrusiveForwardList, InsertAfter2) { +TEST_F(IntrusiveForwardListTest, InsertAfter1) { + InsertAfter1<IFLTestValueList>(); + InsertAfter1<ConstIFLTestValueList>(); + InsertAfter1<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::InsertAfter2() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref; - IntrusiveForwardList<IFLTestValue> ifl; + ListType ifl; auto ref_it = ref.insert_after(ref.before_begin(), { 2, 8, 5 }); - std::vector<IFLTestValue> storage1({ { 2 }, { 8 }, { 5 } }); + std::vector<ValueType> storage1({ { 2 }, { 8 }, { 5 } }); auto ifl_it = ifl.insert_after(ifl.before_begin(), storage1.begin(), storage1.end()); ASSERT_LISTS_EQUAL(ref, ifl); ASSERT_EQ(*ref_it, *ifl_it); - std::vector<IFLTestValue> storage2({ { 7 }, { 2 } }); + std::vector<ValueType> storage2({ { 7 }, { 2 } }); ref_it = ref.insert_after(ref.begin(), { 7, 2 }); ifl_it = ifl.insert_after(ifl.begin(), storage2.begin(), storage2.end()); ASSERT_LISTS_EQUAL(ref, ifl); ASSERT_EQ(*ref_it, *ifl_it); - std::vector<IFLTestValue> storage3({ { 1 }, { 3 }, { 4 }, { 9 } }); + std::vector<ValueType> storage3({ { 1 }, { 3 }, { 4 }, { 9 } }); ref_it = ref.begin(); ifl_it = ifl.begin(); std::advance(ref_it, std::distance(ref.begin(), ref.end()) - 1); @@ -175,10 +300,18 @@ TEST(IntrusiveForwardList, InsertAfter2) { ASSERT_LISTS_EQUAL(ref, ifl); } -TEST(IntrusiveForwardList, EraseAfter1) { +TEST_F(IntrusiveForwardListTest, InsertAfter2) { + InsertAfter2<IFLTestValueList>(); + InsertAfter2<ConstIFLTestValueList>(); + InsertAfter2<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::EraseAfter1() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 1, 2, 7, 4, 5 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); CHECK_EQ(std::distance(ref.begin(), ref.end()), 5); @@ -230,10 +363,18 @@ TEST(IntrusiveForwardList, EraseAfter1) { ASSERT_TRUE(ifl_it == ifl.begin()); } -TEST(IntrusiveForwardList, EraseAfter2) { +TEST_F(IntrusiveForwardListTest, EraseAfter1) { + EraseAfter1<IFLTestValueList>(); + EraseAfter1<ConstIFLTestValueList>(); + EraseAfter1<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::EraseAfter2() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 1, 2, 7, 4, 5, 3, 2, 8, 9 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); CHECK_EQ(std::distance(ref.begin(), ref.end()), 9); @@ -262,13 +403,21 @@ TEST(IntrusiveForwardList, EraseAfter2) { CHECK_EQ(std::distance(ref.begin(), ref.end()), 0); } -TEST(IntrusiveForwardList, SwapClear) { +TEST_F(IntrusiveForwardListTest, EraseAfter2) { + EraseAfter2<IFLTestValueList>(); + EraseAfter2<ConstIFLTestValueList>(); + EraseAfter2<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::SwapClear() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref1({ 1, 2, 7 }); - std::vector<IFLTestValue> storage1(ref1.begin(), ref1.end()); - IntrusiveForwardList<IFLTestValue> ifl1(storage1.begin(), storage1.end()); + std::vector<ValueType> storage1(ref1.begin(), ref1.end()); + ListType ifl1(storage1.begin(), storage1.end()); std::forward_list<int> ref2({ 3, 8, 6 }); - std::vector<IFLTestValue> storage2(ref2.begin(), ref2.end()); - IntrusiveForwardList<IFLTestValue> ifl2(storage2.begin(), storage2.end()); + std::vector<ValueType> storage2(ref2.begin(), ref2.end()); + ListType ifl2(storage2.begin(), storage2.end()); ASSERT_LISTS_EQUAL(ref1, ifl1); ASSERT_LISTS_EQUAL(ref2, ifl2); ref1.swap(ref2); @@ -289,12 +438,20 @@ TEST(IntrusiveForwardList, SwapClear) { ASSERT_LISTS_EQUAL(ref2, ifl2); } -TEST(IntrusiveForwardList, SpliceAfter) { +TEST_F(IntrusiveForwardListTest, SwapClear) { + SwapClear<IFLTestValueList>(); + SwapClear<ConstIFLTestValueList>(); + SwapClear<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::SpliceAfter() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref1({ 3, 1, 2, 7, 4, 5, 4, 8, 7 }); std::forward_list<int> ref2; - std::vector<IFLTestValue> storage(ref1.begin(), ref1.end()); - IntrusiveForwardList<IFLTestValue> ifl1(storage.begin(), storage.end()); - IntrusiveForwardList<IFLTestValue> ifl2; + std::vector<ValueType> storage(ref1.begin(), ref1.end()); + ListType ifl1(storage.begin(), storage.end()); + ListType ifl2; ASSERT_LISTS_EQUAL(ref1, ifl1); ASSERT_LISTS_EQUAL(ref2, ifl2); @@ -398,10 +555,18 @@ TEST(IntrusiveForwardList, SpliceAfter) { ASSERT_LISTS_EQUAL(check, ifl2); } -TEST(IntrusiveForwardList, Remove) { +TEST_F(IntrusiveForwardListTest, SpliceAfter) { + SpliceAfter<IFLTestValueList>(); + SpliceAfter<ConstIFLTestValueList>(); + SpliceAfter<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::Remove() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 3, 1, 2, 7, 4, 5, 4, 8, 7 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); ref.remove(1); ifl.remove(1); @@ -409,20 +574,28 @@ TEST(IntrusiveForwardList, Remove) { ref.remove(4); ifl.remove(4); ASSERT_LISTS_EQUAL(ref, ifl); - auto odd = [](IFLTestValue value) { return (value.value & 1) != 0; }; // NOLINT(readability/braces) + auto odd = [](ValueType value) { return (value.value & 1) != 0; }; // NOLINT(readability/braces) ref.remove_if(odd); ifl.remove_if(odd); ASSERT_LISTS_EQUAL(ref, ifl); - auto all = [](IFLTestValue value ATTRIBUTE_UNUSED) { return true; }; // NOLINT(readability/braces) + auto all = [](ValueType value ATTRIBUTE_UNUSED) { return true; }; // NOLINT(readability/braces) ref.remove_if(all); ifl.remove_if(all); ASSERT_LISTS_EQUAL(ref, ifl); } -TEST(IntrusiveForwardList, Unique) { +TEST_F(IntrusiveForwardListTest, Remove) { + Remove<IFLTestValueList>(); + Remove<ConstIFLTestValueList>(); + Remove<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::Unique() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 3, 1, 1, 2, 3, 3, 7, 7, 4, 4, 5, 7 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); ref.unique(); ifl.unique(); @@ -430,7 +603,7 @@ TEST(IntrusiveForwardList, Unique) { std::forward_list<int> check({ 3, 1, 2, 3, 7, 4, 5, 7 }); ASSERT_LISTS_EQUAL(check, ifl); - auto bin_pred = [](IFLTestValue lhs, IFLTestValue rhs) { + auto bin_pred = [](const ValueType& lhs, const ValueType& rhs) { return (lhs.value & ~1) == (rhs.value & ~1); }; ref.unique(bin_pred); @@ -440,13 +613,21 @@ TEST(IntrusiveForwardList, Unique) { ASSERT_LISTS_EQUAL(check, ifl); } -TEST(IntrusiveForwardList, Merge) { +TEST_F(IntrusiveForwardListTest, Unique) { + Unique<IFLTestValueList>(); + Unique<ConstIFLTestValueList>(); + Unique<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::Merge() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref1({ 1, 4, 8, 8, 12 }); - std::vector<IFLTestValue> storage1(ref1.begin(), ref1.end()); - IntrusiveForwardList<IFLTestValue> ifl1(storage1.begin(), storage1.end()); + std::vector<ValueType> storage1(ref1.begin(), ref1.end()); + ListType ifl1(storage1.begin(), storage1.end()); std::forward_list<int> ref2({ 3, 5, 6, 7, 9 }); - std::vector<IFLTestValue> storage2(ref2.begin(), ref2.end()); - IntrusiveForwardList<IFLTestValue> ifl2(storage2.begin(), storage2.end()); + std::vector<ValueType> storage2(ref2.begin(), ref2.end()); + ListType ifl2(storage2.begin(), storage2.end()); ASSERT_LISTS_EQUAL(ref1, ifl1); ASSERT_LISTS_EQUAL(ref2, ifl2); CHECK(std::is_sorted(ref1.begin(), ref1.end())); @@ -460,10 +641,18 @@ TEST(IntrusiveForwardList, Merge) { ASSERT_LISTS_EQUAL(check, ifl1); } -TEST(IntrusiveForwardList, Sort1) { +TEST_F(IntrusiveForwardListTest, Merge) { + Merge<IFLTestValueList>(); + Merge<ConstIFLTestValueList>(); + Merge<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::Sort1() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 2, 9, 8, 3, 7, 4, 1, 5, 3, 0 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); CHECK(!std::is_sorted(ref.begin(), ref.end())); ref.sort(); @@ -473,12 +662,20 @@ TEST(IntrusiveForwardList, Sort1) { ASSERT_LISTS_EQUAL(check, ifl); } -TEST(IntrusiveForwardList, Sort2) { +TEST_F(IntrusiveForwardListTest, Sort1) { + Sort1<IFLTestValueList>(); + Sort1<ConstIFLTestValueList>(); + Sort1<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::Sort2() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 2, 9, 8, 3, 7, 4, 1, 5, 3, 0 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); - auto cmp = [](IFLTestValue lhs, IFLTestValue rhs) { + auto cmp = [](const ValueType& lhs, const ValueType& rhs) { return (lhs.value & ~1) < (rhs.value & ~1); }; CHECK(!std::is_sorted(ref.begin(), ref.end(), cmp)); @@ -489,10 +686,18 @@ TEST(IntrusiveForwardList, Sort2) { ASSERT_LISTS_EQUAL(check, ifl); } -TEST(IntrusiveForwardList, Reverse) { +TEST_F(IntrusiveForwardListTest, Sort2) { + Sort2<IFLTestValueList>(); + Sort2<ConstIFLTestValueList>(); + Sort2<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::Reverse() { + using ValueType = typename ListType::value_type; std::forward_list<int> ref({ 8, 3, 5, 4, 1, 3 }); - std::vector<IFLTestValue> storage(ref.begin(), ref.end()); - IntrusiveForwardList<IFLTestValue> ifl(storage.begin(), storage.end()); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); ASSERT_LISTS_EQUAL(ref, ifl); CHECK(!std::is_sorted(ref.begin(), ref.end())); ref.reverse(); @@ -502,4 +707,73 @@ TEST(IntrusiveForwardList, Reverse) { ASSERT_LISTS_EQUAL(check, ifl); } +TEST_F(IntrusiveForwardListTest, Reverse) { + Reverse<IFLTestValueList>(); + Reverse<ConstIFLTestValueList>(); + Reverse<IFLTestValue2List>(); +} + +template <typename ListType> +void IntrusiveForwardListTest::ModifyValue() { + using ValueType = typename ListType::value_type; + std::forward_list<int> ref({ 3, 7, 42 }); + std::vector<ValueType> storage(ref.begin(), ref.end()); + ListType ifl(storage.begin(), storage.end()); + ASSERT_LISTS_EQUAL(ref, ifl); + + auto add1 = [](const ValueType& value) { return value.value + 1; }; // NOLINT [readability/braces] + std::transform(ref.begin(), ref.end(), ref.begin(), add1); + std::transform(ifl.begin(), ifl.end(), ifl.begin(), add1); + ASSERT_LISTS_EQUAL(ref, ifl); +} + +TEST_F(IntrusiveForwardListTest, ModifyValue) { + ModifyValue<IFLTestValueList>(); + // Does not compile with ConstIFLTestValueList because LHS of the assignment is const. + // ModifyValue<ConstIFLTestValueList>(); + static_assert(std::is_const<ConstIFLTestValueList::iterator::value_type>::value, "Const check."); + ModifyValue<IFLTestValue2List>(); +} + +struct Tag1; +struct Tag2; +struct TwoListsValue : public IntrusiveForwardListNode<TwoListsValue, Tag1>, + public IntrusiveForwardListNode<TwoListsValue, Tag2> { + // Deliberately not explicit. + TwoListsValue(int v) : value(v) { } // NOLINT(runtime/explicit) + + int value; +}; +using FirstList = + IntrusiveForwardList<TwoListsValue, IntrusiveForwardListBaseHookTraits<TwoListsValue, Tag1>>; +using SecondList = + IntrusiveForwardList<TwoListsValue, IntrusiveForwardListBaseHookTraits<TwoListsValue, Tag2>>; + +bool operator==(const TwoListsValue& lhs, const TwoListsValue& rhs) { + return lhs.value == rhs.value; +} + +TEST_F(IntrusiveForwardListTest, TwoLists) { + // Test that a value can be in two lists at the same time and the hooks do not interfere. + std::vector<TwoListsValue> storage({ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); // storage[i] = i + + std::vector<int> order1({ 3, 1, 7, 2, 8, 9, 4, 0, 6, 5 }); + FirstList list1; + auto pos1 = list1.before_begin(); + for (size_t idx : order1) { + pos1 = list1.insert_after(pos1, storage[idx]); + } + + std::vector<int> order2({ 8, 5, 1, 6, 7, 2, 9, 3, 0, 4 }); + SecondList list2; + auto pos2 = list2.before_begin(); + for (size_t idx : order2) { + pos2 = list2.insert_after(pos2, storage[idx]); + } + + // Using `storage[i] = i`, we can easily compare that nodes of each list are in the right order. + ASSERT_LISTS_EQUAL(order1, list1); + ASSERT_LISTS_EQUAL(order2, list2); +} + } // namespace art diff --git a/compiler/utils/label.h b/compiler/utils/label.h index 4c6ae8e218..85710d0811 100644 --- a/compiler/utils/label.h +++ b/compiler/utils/label.h @@ -26,10 +26,6 @@ class Assembler; class AssemblerBuffer; class AssemblerFixup; -namespace arm { - class ArmAssembler; - class Thumb2Assembler; -} // namespace arm namespace arm64 { class Arm64Assembler; } // namespace arm64 @@ -116,8 +112,6 @@ class Label { CHECK(IsLinked()); } - friend class arm::ArmAssembler; - friend class arm::Thumb2Assembler; friend class arm64::Arm64Assembler; friend class mips::MipsAssembler; friend class mips64::Mips64Assembler; diff --git a/compiler/utils/mips/assembler_mips.cc b/compiler/utils/mips/assembler_mips.cc index a99d02d4d0..c581f1c58f 100644 --- a/compiler/utils/mips/assembler_mips.cc +++ b/compiler/utils/mips/assembler_mips.cc @@ -404,6 +404,129 @@ uint32_t MipsAssembler::EmitFI(int opcode, int fmt, FRegister ft, uint16_t imm) return encoding; } +uint32_t MipsAssembler::EmitMsa3R(int operation, + int df, + VectorRegister wt, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(wt, kNoVectorRegister); + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaOperationShift | + df << kDfShift | + static_cast<uint32_t>(wt) << kWtShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaBIT(int operation, + int df_m, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaOperationShift | + df_m << kDfMShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaELM(int operation, + int df_n, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaELMOperationShift | + df_n << kDfNShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaMI10(int s10, + Register rs, + VectorRegister wd, + int minor_opcode, + int df) { + CHECK_NE(rs, kNoRegister); + CHECK_NE(wd, kNoVectorRegister); + CHECK(IsUint<10>(s10)) << s10; + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + s10 << kS10Shift | + static_cast<uint32_t>(rs) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode << kS10MinorShift | + df; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaI10(int operation, + int df, + int i10, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(wd, kNoVectorRegister); + CHECK(IsUint<10>(i10)) << i10; + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaOperationShift | + df << kDfShift | + i10 << kI10Shift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsa2R(int operation, + int df, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsa2ROperationShift | + df << kDf2RShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsa2RF(int operation, + int df, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsa2RFOperationShift | + df << kDf2RShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + void MipsAssembler::Addu(Register rd, Register rs, Register rt) { DsFsmInstrRrr(EmitR(0, rs, rt, rd, 0, 0x21), rd, rs, rt); } @@ -635,9 +758,8 @@ void MipsAssembler::Ins(Register rd, Register rt, int pos, int size) { DsFsmInstrRrr(EmitR(0x1f, rt, rd, static_cast<Register>(pos + size - 1), pos, 0x04), rd, rd, rt); } -// TODO: This instruction is available in both R6 and MSA and it should be used when available. void MipsAssembler::Lsa(Register rd, Register rs, Register rt, int saPlusOne) { - CHECK(IsR6()); + CHECK(IsR6() || HasMsa()); CHECK(1 <= saPlusOne && saPlusOne <= 4) << saPlusOne; int sa = saPlusOne - 1; DsFsmInstrRrr(EmitR(0x0, rs, rt, rd, sa, 0x05), rd, rs, rt); @@ -653,7 +775,7 @@ void MipsAssembler::ShiftAndAdd(Register dst, if (shamt == TIMES_1) { // Catch the special case where the shift amount is zero (0). Addu(dst, src_base, src_idx); - } else if (IsR6()) { + } else if (IsR6() || HasMsa()) { Lsa(dst, src_idx, src_base, shamt); } else { Sll(tmp, src_idx, shamt); @@ -1709,6 +1831,1090 @@ void MipsAssembler::PopAndReturn(Register rd, Register rt) { SetReorder(reordering); } +void MipsAssembler::AndV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::OrV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::NorV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::XorV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FaddD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FsubW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FsubD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmulW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmulD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FdivW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x2, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FdivD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x3, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ffint_sW(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19e, 0x0, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::Ffint_sD(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19e, 0x1, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::Ftint_sW(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19c, 0x0, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::Ftint_sD(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19c, 0x1, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SllB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SllH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SllW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SllD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x0, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x1, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x2, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x3, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x0, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x1, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x2, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x3, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SlliB(VectorRegister wd, VectorRegister ws, int shamt3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(shamt3)) << shamt3; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt3 | kMsaDfMByteMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SlliH(VectorRegister wd, VectorRegister ws, int shamt4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(shamt4)) << shamt4; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt4 | kMsaDfMHalfwordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SlliW(VectorRegister wd, VectorRegister ws, int shamt5) { + CHECK(HasMsa()); + CHECK(IsUint<5>(shamt5)) << shamt5; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt5 | kMsaDfMWordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SlliD(VectorRegister wd, VectorRegister ws, int shamt6) { + CHECK(HasMsa()); + CHECK(IsUint<6>(shamt6)) << shamt6; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt6 | kMsaDfMDoublewordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiB(VectorRegister wd, VectorRegister ws, int shamt3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(shamt3)) << shamt3; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt3 | kMsaDfMByteMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiH(VectorRegister wd, VectorRegister ws, int shamt4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(shamt4)) << shamt4; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt4 | kMsaDfMHalfwordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiW(VectorRegister wd, VectorRegister ws, int shamt5) { + CHECK(HasMsa()); + CHECK(IsUint<5>(shamt5)) << shamt5; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt5 | kMsaDfMWordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiD(VectorRegister wd, VectorRegister ws, int shamt6) { + CHECK(HasMsa()); + CHECK(IsUint<6>(shamt6)) << shamt6; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt6 | kMsaDfMDoublewordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliB(VectorRegister wd, VectorRegister ws, int shamt3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(shamt3)) << shamt3; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt3 | kMsaDfMByteMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliH(VectorRegister wd, VectorRegister ws, int shamt4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(shamt4)) << shamt4; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt4 | kMsaDfMHalfwordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliW(VectorRegister wd, VectorRegister ws, int shamt5) { + CHECK(HasMsa()); + CHECK(IsUint<5>(shamt5)) << shamt5; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt5 | kMsaDfMWordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliD(VectorRegister wd, VectorRegister ws, int shamt6) { + CHECK(HasMsa()); + CHECK(IsUint<6>(shamt6)) << shamt6; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt6 | kMsaDfMDoublewordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::MoveV(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsaBIT(0x1, 0x3e, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiB(VectorRegister wd, VectorRegister ws, int n4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(n4)) << n4; + DsFsmInstrFff(EmitMsaELM(0x1, n4 | kMsaDfNByteMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiH(VectorRegister wd, VectorRegister ws, int n3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(n3)) << n3; + DsFsmInstrFff(EmitMsaELM(0x1, n3 | kMsaDfNHalfwordMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiW(VectorRegister wd, VectorRegister ws, int n2) { + CHECK(HasMsa()); + CHECK(IsUint<2>(n2)) << n2; + DsFsmInstrFff(EmitMsaELM(0x1, n2 | kMsaDfNWordMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiD(VectorRegister wd, VectorRegister ws, int n1) { + CHECK(HasMsa()); + CHECK(IsUint<1>(n1)) << n1; + DsFsmInstrFff(EmitMsaELM(0x1, n1 | kMsaDfNDoublewordMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::FillB(VectorRegister wd, Register rs) { + CHECK(HasMsa()); + DsFsmInstrFr(EmitMsa2R(0xc0, 0x0, static_cast<VectorRegister>(rs), wd, 0x1e), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::FillH(VectorRegister wd, Register rs) { + CHECK(HasMsa()); + DsFsmInstrFr(EmitMsa2R(0xc0, 0x1, static_cast<VectorRegister>(rs), wd, 0x1e), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::FillW(VectorRegister wd, Register rs) { + CHECK(HasMsa()); + DsFsmInstrFr(EmitMsa2R(0xc0, 0x2, static_cast<VectorRegister>(rs), wd, 0x1e), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdiB(VectorRegister wd, int imm8) { + CHECK(HasMsa()); + CHECK(IsInt<8>(imm8)) << imm8; + DsFsmInstrFr(EmitMsaI10(0x6, 0x0, imm8 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdiH(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + DsFsmInstrFr(EmitMsaI10(0x6, 0x1, imm10 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdiW(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + DsFsmInstrFr(EmitMsaI10(0x6, 0x2, imm10 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdiD(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + DsFsmInstrFr(EmitMsaI10(0x6, 0x3, imm10 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdB(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<10>(offset)) << offset; + DsFsmInstrFr(EmitMsaMI10(offset & kMsaS10Mask, rs, wd, 0x8, 0x0), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdH(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<11>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsHalfwordSize); + DsFsmInstrFr(EmitMsaMI10((offset >> TIMES_2) & kMsaS10Mask, rs, wd, 0x8, 0x1), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdW(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<12>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsWordSize); + DsFsmInstrFr(EmitMsaMI10((offset >> TIMES_4) & kMsaS10Mask, rs, wd, 0x8, 0x2), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdD(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<13>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsDoublewordSize); + DsFsmInstrFr(EmitMsaMI10((offset >> TIMES_8) & kMsaS10Mask, rs, wd, 0x8, 0x3), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::StB(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<10>(offset)) << offset; + DsFsmInstrFR(EmitMsaMI10(offset & kMsaS10Mask, rs, wd, 0x9, 0x0), static_cast<FRegister>(wd), rs); +} + +void MipsAssembler::StH(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<11>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsHalfwordSize); + DsFsmInstrFR(EmitMsaMI10((offset >> TIMES_2) & kMsaS10Mask, rs, wd, 0x9, 0x1), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::StW(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<12>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsWordSize); + DsFsmInstrFR(EmitMsaMI10((offset >> TIMES_4) & kMsaS10Mask, rs, wd, 0x9, 0x2), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::StD(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<13>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsDoublewordSize); + DsFsmInstrFR(EmitMsaMI10((offset >> TIMES_8) & kMsaS10Mask, rs, wd, 0x9, 0x3), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::IlvrB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::IlvrH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::IlvrW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::IlvrD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::ReplicateFPToVectorRegister(VectorRegister dst, + FRegister src, + bool is_double) { + // Float or double in FPU register Fx can be considered as 0th element in vector register Wx. + if (is_double) { + SplatiD(dst, static_cast<VectorRegister>(src), 0); + } else { + SplatiW(dst, static_cast<VectorRegister>(src), 0); + } +} + void MipsAssembler::LoadConst32(Register rd, int32_t value) { if (IsUint<16>(value)) { // Use OR with (unsigned) immediate to encode 16b unsigned int. @@ -3245,6 +4451,106 @@ void MipsAssembler::AdjustBaseAndOffset(Register& base, CHECK_EQ(misalignment, offset & (kMipsDoublewordSize - 1)); } +void MipsAssembler::AdjustBaseOffsetAndElementSizeShift(Register& base, + int32_t& offset, + int& element_size_shift) { + // This method is used to adjust the base register, offset and element_size_shift + // for a vector load/store when the offset doesn't fit into allowed number of bits. + // MSA ld.df and st.df instructions take signed offsets as arguments, but maximum + // offset is dependant on the size of the data format df (10-bit offsets for ld.b, + // 11-bit for ld.h, 12-bit for ld.w and 13-bit for ld.d). + // If element_size_shift is non-negative at entry, it won't be changed, but offset + // will be checked for appropriate alignment. If negative at entry, it will be + // adjusted based on offset for maximum fit. + // It's assumed that `base` is a multiple of 8. + CHECK_NE(base, AT); // Must not overwrite the register `base` while loading `offset`. + + if (element_size_shift >= 0) { + CHECK_LE(element_size_shift, TIMES_8); + CHECK_GE(JAVASTYLE_CTZ(offset), element_size_shift); + } else if (IsAligned<kMipsDoublewordSize>(offset)) { + element_size_shift = TIMES_8; + } else if (IsAligned<kMipsWordSize>(offset)) { + element_size_shift = TIMES_4; + } else if (IsAligned<kMipsHalfwordSize>(offset)) { + element_size_shift = TIMES_2; + } else { + element_size_shift = TIMES_1; + } + + const int low_len = 10 + element_size_shift; // How many low bits of `offset` ld.df/st.df + // will take. + int16_t low = offset & ((1 << low_len) - 1); // Isolate these bits. + low -= (low & (1 << (low_len - 1))) << 1; // Sign-extend these bits. + if (low == offset) { + return; // `offset` fits into ld.df/st.df. + } + + // First, see if `offset` can be represented as a sum of two or three signed offsets. + // This can save an instruction or two. + + // Max int16_t that's a multiple of element size. + const int32_t kMaxDeltaForSimpleAdjustment = 0x8000 - (1 << element_size_shift); + // Max ld.df/st.df offset that's a multiple of element size. + const int32_t kMaxLoadStoreOffset = 0x1ff << element_size_shift; + const int32_t kMaxOffsetForSimpleAdjustment = kMaxDeltaForSimpleAdjustment + kMaxLoadStoreOffset; + const int32_t kMinOffsetForMediumAdjustment = 2 * kMaxDeltaForSimpleAdjustment; + const int32_t kMaxOffsetForMediumAdjustment = kMinOffsetForMediumAdjustment + kMaxLoadStoreOffset; + + if (IsInt<16>(offset)) { + Addiu(AT, base, offset); + offset = 0; + } else if (0 <= offset && offset <= kMaxOffsetForSimpleAdjustment) { + Addiu(AT, base, kMaxDeltaForSimpleAdjustment); + offset -= kMaxDeltaForSimpleAdjustment; + } else if (-kMaxOffsetForSimpleAdjustment <= offset && offset < 0) { + Addiu(AT, base, -kMaxDeltaForSimpleAdjustment); + offset += kMaxDeltaForSimpleAdjustment; + } else if (!IsR6() && 0 <= offset && offset <= kMaxOffsetForMediumAdjustment) { + Addiu(AT, base, kMaxDeltaForSimpleAdjustment); + if (offset <= kMinOffsetForMediumAdjustment) { + Addiu(AT, AT, offset - kMaxDeltaForSimpleAdjustment); + offset = 0; + } else { + Addiu(AT, AT, kMaxDeltaForSimpleAdjustment); + offset -= kMinOffsetForMediumAdjustment; + } + } else if (!IsR6() && -kMaxOffsetForMediumAdjustment <= offset && offset < 0) { + Addiu(AT, base, -kMaxDeltaForSimpleAdjustment); + if (-kMinOffsetForMediumAdjustment <= offset) { + Addiu(AT, AT, offset + kMaxDeltaForSimpleAdjustment); + offset = 0; + } else { + Addiu(AT, AT, -kMaxDeltaForSimpleAdjustment); + offset += kMinOffsetForMediumAdjustment; + } + } else { + // 16-bit or smaller parts of `offset`: + // |31 hi 16|15 mid 13-10|12-9 low 0| + // + // Instructions that supply each part as a signed integer addend: + // |aui |addiu |ld.df/st.df | + uint32_t tmp = static_cast<uint32_t>(offset) - low; // Exclude `low` from the rest of `offset` + // (accounts for sign of `low`). + tmp += (tmp & (UINT32_C(1) << 15)) << 1; // Account for sign extension in addiu. + int16_t mid = Low16Bits(tmp); + int16_t hi = High16Bits(tmp); + if (IsR6()) { + Aui(AT, base, hi); + } else { + Lui(AT, hi); + Addu(AT, AT, base); + } + if (mid != 0) { + Addiu(AT, AT, mid); + } + offset = low; + } + base = AT; + CHECK_GE(JAVASTYLE_CTZ(offset), element_size_shift); + CHECK(IsInt<10>(offset >> element_size_shift)); +} + void MipsAssembler::LoadFromOffset(LoadOperandType type, Register reg, Register base, @@ -3260,6 +4566,10 @@ void MipsAssembler::LoadDFromOffset(FRegister reg, Register base, int32_t offset LoadDFromOffset<>(reg, base, offset); } +void MipsAssembler::LoadQFromOffset(FRegister reg, Register base, int32_t offset) { + LoadQFromOffset<>(reg, base, offset); +} + void MipsAssembler::EmitLoad(ManagedRegister m_dst, Register src_register, int32_t src_offset, size_t size) { MipsManagedRegister dst = m_dst.AsMips(); @@ -3299,6 +4609,10 @@ void MipsAssembler::StoreDToOffset(FRegister reg, Register base, int32_t offset) StoreDToOffset<>(reg, base, offset); } +void MipsAssembler::StoreQToOffset(FRegister reg, Register base, int32_t offset) { + StoreQToOffset<>(reg, base, offset); +} + static dwarf::Reg DWARFReg(Register reg) { return dwarf::Reg::MipsCore(static_cast<int>(reg)); } diff --git a/compiler/utils/mips/assembler_mips.h b/compiler/utils/mips/assembler_mips.h index 463daeb5d7..33803bb576 100644 --- a/compiler/utils/mips/assembler_mips.h +++ b/compiler/utils/mips/assembler_mips.h @@ -25,6 +25,7 @@ #include "base/arena_containers.h" #include "base/enums.h" #include "base/macros.h" +#include "base/stl_util_identity.h" #include "constants_mips.h" #include "globals.h" #include "managed_register_mips.h" @@ -36,6 +37,7 @@ namespace art { namespace mips { +static constexpr size_t kMipsHalfwordSize = 2; static constexpr size_t kMipsWordSize = 4; static constexpr size_t kMipsDoublewordSize = 8; @@ -45,14 +47,16 @@ enum LoadOperandType { kLoadSignedHalfword, kLoadUnsignedHalfword, kLoadWord, - kLoadDoubleword + kLoadDoubleword, + kLoadQuadword }; enum StoreOperandType { kStoreByte, kStoreHalfword, kStoreWord, - kStoreDoubleword + kStoreDoubleword, + kStoreQuadword }; // Used to test the values returned by ClassS/ClassD. @@ -194,6 +198,7 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi last_position_adjustment_(0), last_old_position_(0), last_branch_id_(0), + has_msa_(instruction_set_features != nullptr ? instruction_set_features->HasMsa() : false), isa_features_(instruction_set_features) { cfi().DelayEmittingAdvancePCs(); } @@ -464,6 +469,152 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi void Clear(Register rd); void Not(Register rd, Register rs); + // MSA instructions. + void AndV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void OrV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void NorV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void XorV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + void AddvB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void AddvH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void AddvW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void AddvD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + void FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FaddD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FsubW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FsubD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmulW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmulD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FdivW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FdivD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + void Ffint_sW(VectorRegister wd, VectorRegister ws); + void Ffint_sD(VectorRegister wd, VectorRegister ws); + void Ftint_sW(VectorRegister wd, VectorRegister ws); + void Ftint_sD(VectorRegister wd, VectorRegister ws); + + void SllB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SllH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SllW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SllD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + // Immediate shift instructions, where shamtN denotes shift amount (must be between 0 and 2^N-1). + void SlliB(VectorRegister wd, VectorRegister ws, int shamt3); + void SlliH(VectorRegister wd, VectorRegister ws, int shamt4); + void SlliW(VectorRegister wd, VectorRegister ws, int shamt5); + void SlliD(VectorRegister wd, VectorRegister ws, int shamt6); + void SraiB(VectorRegister wd, VectorRegister ws, int shamt3); + void SraiH(VectorRegister wd, VectorRegister ws, int shamt4); + void SraiW(VectorRegister wd, VectorRegister ws, int shamt5); + void SraiD(VectorRegister wd, VectorRegister ws, int shamt6); + void SrliB(VectorRegister wd, VectorRegister ws, int shamt3); + void SrliH(VectorRegister wd, VectorRegister ws, int shamt4); + void SrliW(VectorRegister wd, VectorRegister ws, int shamt5); + void SrliD(VectorRegister wd, VectorRegister ws, int shamt6); + + void MoveV(VectorRegister wd, VectorRegister ws); + void SplatiB(VectorRegister wd, VectorRegister ws, int n4); + void SplatiH(VectorRegister wd, VectorRegister ws, int n3); + void SplatiW(VectorRegister wd, VectorRegister ws, int n2); + void SplatiD(VectorRegister wd, VectorRegister ws, int n1); + void FillB(VectorRegister wd, Register rs); + void FillH(VectorRegister wd, Register rs); + void FillW(VectorRegister wd, Register rs); + + void LdiB(VectorRegister wd, int imm8); + void LdiH(VectorRegister wd, int imm10); + void LdiW(VectorRegister wd, int imm10); + void LdiD(VectorRegister wd, int imm10); + void LdB(VectorRegister wd, Register rs, int offset); + void LdH(VectorRegister wd, Register rs, int offset); + void LdW(VectorRegister wd, Register rs, int offset); + void LdD(VectorRegister wd, Register rs, int offset); + void StB(VectorRegister wd, Register rs, int offset); + void StH(VectorRegister wd, Register rs, int offset); + void StW(VectorRegister wd, Register rs, int offset); + void StD(VectorRegister wd, Register rs, int offset); + + void IlvrB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void IlvrH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void IlvrW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void IlvrD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + // Helper for replicating floating point value in all destination elements. + void ReplicateFPToVectorRegister(VectorRegister dst, FRegister src, bool is_double); + // Higher level composite instructions. void LoadConst32(Register rd, int32_t value); void LoadConst64(Register reg_hi, Register reg_lo, int64_t value); @@ -500,6 +651,9 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi int32_t& offset, bool is_doubleword, bool is_float = false); + void AdjustBaseOffsetAndElementSizeShift(Register& base, + int32_t& offset, + int& element_size_shift); private: // This will be used as an argument for loads/stores @@ -647,6 +801,24 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi } template <typename ImplicitNullChecker = NoImplicitNullChecker> + void LoadQFromOffset(FRegister reg, + Register base, + int32_t offset, + ImplicitNullChecker null_checker = NoImplicitNullChecker()) { + int element_size_shift = -1; + AdjustBaseOffsetAndElementSizeShift(base, offset, element_size_shift); + switch (element_size_shift) { + case TIMES_1: LdB(static_cast<VectorRegister>(reg), base, offset); break; + case TIMES_2: LdH(static_cast<VectorRegister>(reg), base, offset); break; + case TIMES_4: LdW(static_cast<VectorRegister>(reg), base, offset); break; + case TIMES_8: LdD(static_cast<VectorRegister>(reg), base, offset); break; + default: + LOG(FATAL) << "UNREACHABLE"; + } + null_checker(); + } + + template <typename ImplicitNullChecker = NoImplicitNullChecker> void StoreToOffset(StoreOperandType type, Register reg, Register base, @@ -715,12 +887,32 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi } } + template <typename ImplicitNullChecker = NoImplicitNullChecker> + void StoreQToOffset(FRegister reg, + Register base, + int32_t offset, + ImplicitNullChecker null_checker = NoImplicitNullChecker()) { + int element_size_shift = -1; + AdjustBaseOffsetAndElementSizeShift(base, offset, element_size_shift); + switch (element_size_shift) { + case TIMES_1: StB(static_cast<VectorRegister>(reg), base, offset); break; + case TIMES_2: StH(static_cast<VectorRegister>(reg), base, offset); break; + case TIMES_4: StW(static_cast<VectorRegister>(reg), base, offset); break; + case TIMES_8: StD(static_cast<VectorRegister>(reg), base, offset); break; + default: + LOG(FATAL) << "UNREACHABLE"; + } + null_checker(); + } + void LoadFromOffset(LoadOperandType type, Register reg, Register base, int32_t offset); void LoadSFromOffset(FRegister reg, Register base, int32_t offset); void LoadDFromOffset(FRegister reg, Register base, int32_t offset); + void LoadQFromOffset(FRegister reg, Register base, int32_t offset); void StoreToOffset(StoreOperandType type, Register reg, Register base, int32_t offset); void StoreSToOffset(FRegister reg, Register base, int32_t offset); void StoreDToOffset(FRegister reg, Register base, int32_t offset); + void StoreQToOffset(FRegister reg, Register base, int32_t offset); // Emit data (e.g. encoded instruction or immediate) to the instruction stream. void Emit(uint32_t value); @@ -1282,6 +1474,30 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi uint32_t EmitFI(int opcode, int fmt, FRegister rt, uint16_t imm); void EmitBcondR2(BranchCondition cond, Register rs, Register rt, uint16_t imm16); void EmitBcondR6(BranchCondition cond, Register rs, Register rt, uint32_t imm16_21); + uint32_t EmitMsa3R(int operation, + int df, + VectorRegister wt, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); + uint32_t EmitMsaBIT(int operation, + int df_m, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); + uint32_t EmitMsaELM(int operation, + int df_n, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); + uint32_t EmitMsaMI10(int s10, Register rs, VectorRegister wd, int minor_opcode, int df); + uint32_t EmitMsaI10(int operation, int df, int i10, VectorRegister wd, int minor_opcode); + uint32_t EmitMsa2R(int operation, int df, VectorRegister ws, VectorRegister wd, int minor_opcode); + uint32_t EmitMsa2RF(int operation, + int df, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); void Buncond(MipsLabel* label); void Bcond(MipsLabel* label, BranchCondition condition, Register lhs, Register rhs = ZERO); @@ -1332,6 +1548,10 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi // Emits exception block. void EmitExceptionPoll(MipsExceptionSlowPath* exception); + bool HasMsa() const { + return has_msa_; + } + bool IsR6() const { if (isa_features_ != nullptr) { return isa_features_->IsR6(); @@ -1386,6 +1606,8 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi uint32_t last_old_position_; uint32_t last_branch_id_; + const bool has_msa_; + const MipsInstructionSetFeatures* isa_features_; DISALLOW_COPY_AND_ASSIGN(MipsAssembler); diff --git a/compiler/utils/mips/assembler_mips32r5_test.cc b/compiler/utils/mips/assembler_mips32r5_test.cc new file mode 100644 index 0000000000..24b09b5524 --- /dev/null +++ b/compiler/utils/mips/assembler_mips32r5_test.cc @@ -0,0 +1,541 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "assembler_mips.h" + +#include <map> + +#include "base/stl_util.h" +#include "utils/assembler_test.h" + +#define __ GetAssembler()-> + +namespace art { + +struct MIPSCpuRegisterCompare { + bool operator()(const mips::Register& a, const mips::Register& b) const { + return a < b; + } +}; + +class AssemblerMIPS32r5Test : public AssemblerTest<mips::MipsAssembler, + mips::Register, + mips::FRegister, + uint32_t, + mips::VectorRegister> { + public: + typedef AssemblerTest<mips::MipsAssembler, + mips::Register, + mips::FRegister, + uint32_t, + mips::VectorRegister> Base; + + AssemblerMIPS32r5Test() : + instruction_set_features_(MipsInstructionSetFeatures::FromVariant("mips32r5", nullptr)) { + } + + protected: + // Get the typically used name for this architecture, e.g., aarch64, x86-64, ... + std::string GetArchitectureString() OVERRIDE { + return "mips"; + } + + std::string GetAssemblerParameters() OVERRIDE { + return " --no-warn -32 -march=mips32r5 -mmsa"; + } + + void Pad(std::vector<uint8_t>& data) OVERRIDE { + // The GNU linker unconditionally pads the code segment with NOPs to a size that is a multiple + // of 16 and there doesn't appear to be a way to suppress this padding. Our assembler doesn't + // pad, so, in order for two assembler outputs to match, we need to match the padding as well. + // NOP is encoded as four zero bytes on MIPS. + size_t pad_size = RoundUp(data.size(), 16u) - data.size(); + data.insert(data.end(), pad_size, 0); + } + + std::string GetDisassembleParameters() OVERRIDE { + return " -D -bbinary -mmips:isa32r5"; + } + + mips::MipsAssembler* CreateAssembler(ArenaAllocator* arena) OVERRIDE { + return new (arena) mips::MipsAssembler(arena, instruction_set_features_.get()); + } + + void SetUpHelpers() OVERRIDE { + if (registers_.size() == 0) { + registers_.push_back(new mips::Register(mips::ZERO)); + registers_.push_back(new mips::Register(mips::AT)); + registers_.push_back(new mips::Register(mips::V0)); + registers_.push_back(new mips::Register(mips::V1)); + registers_.push_back(new mips::Register(mips::A0)); + registers_.push_back(new mips::Register(mips::A1)); + registers_.push_back(new mips::Register(mips::A2)); + registers_.push_back(new mips::Register(mips::A3)); + registers_.push_back(new mips::Register(mips::T0)); + registers_.push_back(new mips::Register(mips::T1)); + registers_.push_back(new mips::Register(mips::T2)); + registers_.push_back(new mips::Register(mips::T3)); + registers_.push_back(new mips::Register(mips::T4)); + registers_.push_back(new mips::Register(mips::T5)); + registers_.push_back(new mips::Register(mips::T6)); + registers_.push_back(new mips::Register(mips::T7)); + registers_.push_back(new mips::Register(mips::S0)); + registers_.push_back(new mips::Register(mips::S1)); + registers_.push_back(new mips::Register(mips::S2)); + registers_.push_back(new mips::Register(mips::S3)); + registers_.push_back(new mips::Register(mips::S4)); + registers_.push_back(new mips::Register(mips::S5)); + registers_.push_back(new mips::Register(mips::S6)); + registers_.push_back(new mips::Register(mips::S7)); + registers_.push_back(new mips::Register(mips::T8)); + registers_.push_back(new mips::Register(mips::T9)); + registers_.push_back(new mips::Register(mips::K0)); + registers_.push_back(new mips::Register(mips::K1)); + registers_.push_back(new mips::Register(mips::GP)); + registers_.push_back(new mips::Register(mips::SP)); + registers_.push_back(new mips::Register(mips::FP)); + registers_.push_back(new mips::Register(mips::RA)); + + secondary_register_names_.emplace(mips::Register(mips::ZERO), "zero"); + secondary_register_names_.emplace(mips::Register(mips::AT), "at"); + secondary_register_names_.emplace(mips::Register(mips::V0), "v0"); + secondary_register_names_.emplace(mips::Register(mips::V1), "v1"); + secondary_register_names_.emplace(mips::Register(mips::A0), "a0"); + secondary_register_names_.emplace(mips::Register(mips::A1), "a1"); + secondary_register_names_.emplace(mips::Register(mips::A2), "a2"); + secondary_register_names_.emplace(mips::Register(mips::A3), "a3"); + secondary_register_names_.emplace(mips::Register(mips::T0), "t0"); + secondary_register_names_.emplace(mips::Register(mips::T1), "t1"); + secondary_register_names_.emplace(mips::Register(mips::T2), "t2"); + secondary_register_names_.emplace(mips::Register(mips::T3), "t3"); + secondary_register_names_.emplace(mips::Register(mips::T4), "t4"); + secondary_register_names_.emplace(mips::Register(mips::T5), "t5"); + secondary_register_names_.emplace(mips::Register(mips::T6), "t6"); + secondary_register_names_.emplace(mips::Register(mips::T7), "t7"); + secondary_register_names_.emplace(mips::Register(mips::S0), "s0"); + secondary_register_names_.emplace(mips::Register(mips::S1), "s1"); + secondary_register_names_.emplace(mips::Register(mips::S2), "s2"); + secondary_register_names_.emplace(mips::Register(mips::S3), "s3"); + secondary_register_names_.emplace(mips::Register(mips::S4), "s4"); + secondary_register_names_.emplace(mips::Register(mips::S5), "s5"); + secondary_register_names_.emplace(mips::Register(mips::S6), "s6"); + secondary_register_names_.emplace(mips::Register(mips::S7), "s7"); + secondary_register_names_.emplace(mips::Register(mips::T8), "t8"); + secondary_register_names_.emplace(mips::Register(mips::T9), "t9"); + secondary_register_names_.emplace(mips::Register(mips::K0), "k0"); + secondary_register_names_.emplace(mips::Register(mips::K1), "k1"); + secondary_register_names_.emplace(mips::Register(mips::GP), "gp"); + secondary_register_names_.emplace(mips::Register(mips::SP), "sp"); + secondary_register_names_.emplace(mips::Register(mips::FP), "fp"); + secondary_register_names_.emplace(mips::Register(mips::RA), "ra"); + + fp_registers_.push_back(new mips::FRegister(mips::F0)); + fp_registers_.push_back(new mips::FRegister(mips::F1)); + fp_registers_.push_back(new mips::FRegister(mips::F2)); + fp_registers_.push_back(new mips::FRegister(mips::F3)); + fp_registers_.push_back(new mips::FRegister(mips::F4)); + fp_registers_.push_back(new mips::FRegister(mips::F5)); + fp_registers_.push_back(new mips::FRegister(mips::F6)); + fp_registers_.push_back(new mips::FRegister(mips::F7)); + fp_registers_.push_back(new mips::FRegister(mips::F8)); + fp_registers_.push_back(new mips::FRegister(mips::F9)); + fp_registers_.push_back(new mips::FRegister(mips::F10)); + fp_registers_.push_back(new mips::FRegister(mips::F11)); + fp_registers_.push_back(new mips::FRegister(mips::F12)); + fp_registers_.push_back(new mips::FRegister(mips::F13)); + fp_registers_.push_back(new mips::FRegister(mips::F14)); + fp_registers_.push_back(new mips::FRegister(mips::F15)); + fp_registers_.push_back(new mips::FRegister(mips::F16)); + fp_registers_.push_back(new mips::FRegister(mips::F17)); + fp_registers_.push_back(new mips::FRegister(mips::F18)); + fp_registers_.push_back(new mips::FRegister(mips::F19)); + fp_registers_.push_back(new mips::FRegister(mips::F20)); + fp_registers_.push_back(new mips::FRegister(mips::F21)); + fp_registers_.push_back(new mips::FRegister(mips::F22)); + fp_registers_.push_back(new mips::FRegister(mips::F23)); + fp_registers_.push_back(new mips::FRegister(mips::F24)); + fp_registers_.push_back(new mips::FRegister(mips::F25)); + fp_registers_.push_back(new mips::FRegister(mips::F26)); + fp_registers_.push_back(new mips::FRegister(mips::F27)); + fp_registers_.push_back(new mips::FRegister(mips::F28)); + fp_registers_.push_back(new mips::FRegister(mips::F29)); + fp_registers_.push_back(new mips::FRegister(mips::F30)); + fp_registers_.push_back(new mips::FRegister(mips::F31)); + + vec_registers_.push_back(new mips::VectorRegister(mips::W0)); + vec_registers_.push_back(new mips::VectorRegister(mips::W1)); + vec_registers_.push_back(new mips::VectorRegister(mips::W2)); + vec_registers_.push_back(new mips::VectorRegister(mips::W3)); + vec_registers_.push_back(new mips::VectorRegister(mips::W4)); + vec_registers_.push_back(new mips::VectorRegister(mips::W5)); + vec_registers_.push_back(new mips::VectorRegister(mips::W6)); + vec_registers_.push_back(new mips::VectorRegister(mips::W7)); + vec_registers_.push_back(new mips::VectorRegister(mips::W8)); + vec_registers_.push_back(new mips::VectorRegister(mips::W9)); + vec_registers_.push_back(new mips::VectorRegister(mips::W10)); + vec_registers_.push_back(new mips::VectorRegister(mips::W11)); + vec_registers_.push_back(new mips::VectorRegister(mips::W12)); + vec_registers_.push_back(new mips::VectorRegister(mips::W13)); + vec_registers_.push_back(new mips::VectorRegister(mips::W14)); + vec_registers_.push_back(new mips::VectorRegister(mips::W15)); + vec_registers_.push_back(new mips::VectorRegister(mips::W16)); + vec_registers_.push_back(new mips::VectorRegister(mips::W17)); + vec_registers_.push_back(new mips::VectorRegister(mips::W18)); + vec_registers_.push_back(new mips::VectorRegister(mips::W19)); + vec_registers_.push_back(new mips::VectorRegister(mips::W20)); + vec_registers_.push_back(new mips::VectorRegister(mips::W21)); + vec_registers_.push_back(new mips::VectorRegister(mips::W22)); + vec_registers_.push_back(new mips::VectorRegister(mips::W23)); + vec_registers_.push_back(new mips::VectorRegister(mips::W24)); + vec_registers_.push_back(new mips::VectorRegister(mips::W25)); + vec_registers_.push_back(new mips::VectorRegister(mips::W26)); + vec_registers_.push_back(new mips::VectorRegister(mips::W27)); + vec_registers_.push_back(new mips::VectorRegister(mips::W28)); + vec_registers_.push_back(new mips::VectorRegister(mips::W29)); + vec_registers_.push_back(new mips::VectorRegister(mips::W30)); + vec_registers_.push_back(new mips::VectorRegister(mips::W31)); + } + } + + void TearDown() OVERRIDE { + AssemblerTest::TearDown(); + STLDeleteElements(®isters_); + STLDeleteElements(&fp_registers_); + STLDeleteElements(&vec_registers_); + } + + std::vector<mips::Register*> GetRegisters() OVERRIDE { + return registers_; + } + + std::vector<mips::FRegister*> GetFPRegisters() OVERRIDE { + return fp_registers_; + } + + std::vector<mips::VectorRegister*> GetVectorRegisters() OVERRIDE { + return vec_registers_; + } + + uint32_t CreateImmediate(int64_t imm_value) OVERRIDE { + return imm_value; + } + + std::string GetSecondaryRegisterName(const mips::Register& reg) OVERRIDE { + CHECK(secondary_register_names_.find(reg) != secondary_register_names_.end()); + return secondary_register_names_[reg]; + } + + std::string RepeatInsn(size_t count, const std::string& insn) { + std::string result; + for (; count != 0u; --count) { + result += insn; + } + return result; + } + + private: + std::vector<mips::Register*> registers_; + std::map<mips::Register, std::string, MIPSCpuRegisterCompare> secondary_register_names_; + + std::vector<mips::FRegister*> fp_registers_; + std::vector<mips::VectorRegister*> vec_registers_; + std::unique_ptr<const MipsInstructionSetFeatures> instruction_set_features_; +}; + +TEST_F(AssemblerMIPS32r5Test, Toolchain) { + EXPECT_TRUE(CheckTools()); +} + +TEST_F(AssemblerMIPS32r5Test, LoadQFromOffset) { + __ LoadQFromOffset(mips::F0, mips::A0, 0); + __ LoadQFromOffset(mips::F0, mips::A0, 1); + __ LoadQFromOffset(mips::F0, mips::A0, 2); + __ LoadQFromOffset(mips::F0, mips::A0, 4); + __ LoadQFromOffset(mips::F0, mips::A0, 8); + __ LoadQFromOffset(mips::F0, mips::A0, 511); + __ LoadQFromOffset(mips::F0, mips::A0, 512); + __ LoadQFromOffset(mips::F0, mips::A0, 513); + __ LoadQFromOffset(mips::F0, mips::A0, 514); + __ LoadQFromOffset(mips::F0, mips::A0, 516); + __ LoadQFromOffset(mips::F0, mips::A0, 1022); + __ LoadQFromOffset(mips::F0, mips::A0, 1024); + __ LoadQFromOffset(mips::F0, mips::A0, 1025); + __ LoadQFromOffset(mips::F0, mips::A0, 1026); + __ LoadQFromOffset(mips::F0, mips::A0, 1028); + __ LoadQFromOffset(mips::F0, mips::A0, 2044); + __ LoadQFromOffset(mips::F0, mips::A0, 2048); + __ LoadQFromOffset(mips::F0, mips::A0, 2049); + __ LoadQFromOffset(mips::F0, mips::A0, 2050); + __ LoadQFromOffset(mips::F0, mips::A0, 2052); + __ LoadQFromOffset(mips::F0, mips::A0, 4088); + __ LoadQFromOffset(mips::F0, mips::A0, 4096); + __ LoadQFromOffset(mips::F0, mips::A0, 4097); + __ LoadQFromOffset(mips::F0, mips::A0, 4098); + __ LoadQFromOffset(mips::F0, mips::A0, 4100); + __ LoadQFromOffset(mips::F0, mips::A0, 4104); + __ LoadQFromOffset(mips::F0, mips::A0, 0x7FFC); + __ LoadQFromOffset(mips::F0, mips::A0, 0x8000); + __ LoadQFromOffset(mips::F0, mips::A0, 0x10000); + __ LoadQFromOffset(mips::F0, mips::A0, 0x12345678); + __ LoadQFromOffset(mips::F0, mips::A0, 0x12350078); + __ LoadQFromOffset(mips::F0, mips::A0, -256); + __ LoadQFromOffset(mips::F0, mips::A0, -511); + __ LoadQFromOffset(mips::F0, mips::A0, -513); + __ LoadQFromOffset(mips::F0, mips::A0, -1022); + __ LoadQFromOffset(mips::F0, mips::A0, -1026); + __ LoadQFromOffset(mips::F0, mips::A0, -2044); + __ LoadQFromOffset(mips::F0, mips::A0, -2052); + __ LoadQFromOffset(mips::F0, mips::A0, -4096); + __ LoadQFromOffset(mips::F0, mips::A0, -4104); + __ LoadQFromOffset(mips::F0, mips::A0, -32768); + __ LoadQFromOffset(mips::F0, mips::A0, -36856); + __ LoadQFromOffset(mips::F0, mips::A0, 36856); + __ LoadQFromOffset(mips::F0, mips::A0, -69608); + __ LoadQFromOffset(mips::F0, mips::A0, 69608); + __ LoadQFromOffset(mips::F0, mips::A0, 0xABCDEF00); + __ LoadQFromOffset(mips::F0, mips::A0, 0x7FFFABCD); + + const char* expected = + "ld.d $w0, 0($a0)\n" + "ld.b $w0, 1($a0)\n" + "ld.h $w0, 2($a0)\n" + "ld.w $w0, 4($a0)\n" + "ld.d $w0, 8($a0)\n" + "ld.b $w0, 511($a0)\n" + "ld.d $w0, 512($a0)\n" + "addiu $at, $a0, 513\n" + "ld.b $w0, 0($at)\n" + "ld.h $w0, 514($a0)\n" + "ld.w $w0, 516($a0)\n" + "ld.h $w0, 1022($a0)\n" + "ld.d $w0, 1024($a0)\n" + "addiu $at, $a0, 1025\n" + "ld.b $w0, 0($at)\n" + "addiu $at, $a0, 1026\n" + "ld.h $w0, 0($at)\n" + "ld.w $w0, 1028($a0)\n" + "ld.w $w0, 2044($a0)\n" + "ld.d $w0, 2048($a0)\n" + "addiu $at, $a0, 2049\n" + "ld.b $w0, 0($at)\n" + "addiu $at, $a0, 2050\n" + "ld.h $w0, 0($at)\n" + "addiu $at, $a0, 2052\n" + "ld.w $w0, 0($at)\n" + "ld.d $w0, 4088($a0)\n" + "addiu $at, $a0, 4096\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, 4097\n" + "ld.b $w0, 0($at)\n" + "addiu $at, $a0, 4098\n" + "ld.h $w0, 0($at)\n" + "addiu $at, $a0, 4100\n" + "ld.w $w0, 0($at)\n" + "addiu $at, $a0, 4104\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, 0x7FFC\n" + "ld.w $w0, 0($at)\n" + "addiu $at, $a0, 0x7FF8\n" + "ld.d $w0, 8($at)\n" + "addiu $at, $a0, 32760\n" + "addiu $at, $at, 32760\n" + "ld.d $w0, 16($at)\n" + "lui $at, 4660\n" + "addu $at, $at, $a0\n" + "addiu $at, $at, 24576\n" + "ld.d $w0, -2440($at) # 0xF678\n" + "lui $at, 4661\n" + "addu $at, $at, $a0\n" + "ld.d $w0, 120($at)\n" + "ld.d $w0, -256($a0)\n" + "ld.b $w0, -511($a0)\n" + "addiu $at, $a0, -513\n" + "ld.b $w0, 0($at)\n" + "ld.h $w0, -1022($a0)\n" + "addiu $at, $a0, -1026\n" + "ld.h $w0, 0($at)\n" + "ld.w $w0, -2044($a0)\n" + "addiu $at, $a0, -2052\n" + "ld.w $w0, 0($at)\n" + "ld.d $w0, -4096($a0)\n" + "addiu $at, $a0, -4104\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, -32768\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, -32760\n" + "addiu $at, $at, -4096\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, 32760\n" + "addiu $at, $at, 4096\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, -32760\n" + "addiu $at, $at, -32760\n" + "ld.d $w0, -4088($at)\n" + "addiu $at, $a0, 32760\n" + "addiu $at, $at, 32760\n" + "ld.d $w0, 4088($at)\n" + "lui $at, 0xABCE\n" + "addu $at, $at, $a0\n" + "addiu $at, $at, -8192 # 0xE000\n" + "ld.d $w0, 0xF00($at)\n" + "lui $at, 0x8000\n" + "addu $at, $at, $a0\n" + "addiu $at, $at, -21504 # 0xAC00\n" + "ld.b $w0, -51($at) # 0xFFCD\n"; + DriverStr(expected, "LoadQFromOffset"); +} + +TEST_F(AssemblerMIPS32r5Test, StoreQToOffset) { + __ StoreQToOffset(mips::F0, mips::A0, 0); + __ StoreQToOffset(mips::F0, mips::A0, 1); + __ StoreQToOffset(mips::F0, mips::A0, 2); + __ StoreQToOffset(mips::F0, mips::A0, 4); + __ StoreQToOffset(mips::F0, mips::A0, 8); + __ StoreQToOffset(mips::F0, mips::A0, 511); + __ StoreQToOffset(mips::F0, mips::A0, 512); + __ StoreQToOffset(mips::F0, mips::A0, 513); + __ StoreQToOffset(mips::F0, mips::A0, 514); + __ StoreQToOffset(mips::F0, mips::A0, 516); + __ StoreQToOffset(mips::F0, mips::A0, 1022); + __ StoreQToOffset(mips::F0, mips::A0, 1024); + __ StoreQToOffset(mips::F0, mips::A0, 1025); + __ StoreQToOffset(mips::F0, mips::A0, 1026); + __ StoreQToOffset(mips::F0, mips::A0, 1028); + __ StoreQToOffset(mips::F0, mips::A0, 2044); + __ StoreQToOffset(mips::F0, mips::A0, 2048); + __ StoreQToOffset(mips::F0, mips::A0, 2049); + __ StoreQToOffset(mips::F0, mips::A0, 2050); + __ StoreQToOffset(mips::F0, mips::A0, 2052); + __ StoreQToOffset(mips::F0, mips::A0, 4088); + __ StoreQToOffset(mips::F0, mips::A0, 4096); + __ StoreQToOffset(mips::F0, mips::A0, 4097); + __ StoreQToOffset(mips::F0, mips::A0, 4098); + __ StoreQToOffset(mips::F0, mips::A0, 4100); + __ StoreQToOffset(mips::F0, mips::A0, 4104); + __ StoreQToOffset(mips::F0, mips::A0, 0x7FFC); + __ StoreQToOffset(mips::F0, mips::A0, 0x8000); + __ StoreQToOffset(mips::F0, mips::A0, 0x10000); + __ StoreQToOffset(mips::F0, mips::A0, 0x12345678); + __ StoreQToOffset(mips::F0, mips::A0, 0x12350078); + __ StoreQToOffset(mips::F0, mips::A0, -256); + __ StoreQToOffset(mips::F0, mips::A0, -511); + __ StoreQToOffset(mips::F0, mips::A0, -513); + __ StoreQToOffset(mips::F0, mips::A0, -1022); + __ StoreQToOffset(mips::F0, mips::A0, -1026); + __ StoreQToOffset(mips::F0, mips::A0, -2044); + __ StoreQToOffset(mips::F0, mips::A0, -2052); + __ StoreQToOffset(mips::F0, mips::A0, -4096); + __ StoreQToOffset(mips::F0, mips::A0, -4104); + __ StoreQToOffset(mips::F0, mips::A0, -32768); + __ StoreQToOffset(mips::F0, mips::A0, -36856); + __ StoreQToOffset(mips::F0, mips::A0, 36856); + __ StoreQToOffset(mips::F0, mips::A0, -69608); + __ StoreQToOffset(mips::F0, mips::A0, 69608); + __ StoreQToOffset(mips::F0, mips::A0, 0xABCDEF00); + __ StoreQToOffset(mips::F0, mips::A0, 0x7FFFABCD); + + const char* expected = + "st.d $w0, 0($a0)\n" + "st.b $w0, 1($a0)\n" + "st.h $w0, 2($a0)\n" + "st.w $w0, 4($a0)\n" + "st.d $w0, 8($a0)\n" + "st.b $w0, 511($a0)\n" + "st.d $w0, 512($a0)\n" + "addiu $at, $a0, 513\n" + "st.b $w0, 0($at)\n" + "st.h $w0, 514($a0)\n" + "st.w $w0, 516($a0)\n" + "st.h $w0, 1022($a0)\n" + "st.d $w0, 1024($a0)\n" + "addiu $at, $a0, 1025\n" + "st.b $w0, 0($at)\n" + "addiu $at, $a0, 1026\n" + "st.h $w0, 0($at)\n" + "st.w $w0, 1028($a0)\n" + "st.w $w0, 2044($a0)\n" + "st.d $w0, 2048($a0)\n" + "addiu $at, $a0, 2049\n" + "st.b $w0, 0($at)\n" + "addiu $at, $a0, 2050\n" + "st.h $w0, 0($at)\n" + "addiu $at, $a0, 2052\n" + "st.w $w0, 0($at)\n" + "st.d $w0, 4088($a0)\n" + "addiu $at, $a0, 4096\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, 4097\n" + "st.b $w0, 0($at)\n" + "addiu $at, $a0, 4098\n" + "st.h $w0, 0($at)\n" + "addiu $at, $a0, 4100\n" + "st.w $w0, 0($at)\n" + "addiu $at, $a0, 4104\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, 0x7FFC\n" + "st.w $w0, 0($at)\n" + "addiu $at, $a0, 0x7FF8\n" + "st.d $w0, 8($at)\n" + "addiu $at, $a0, 32760\n" + "addiu $at, $at, 32760\n" + "st.d $w0, 16($at)\n" + "lui $at, 4660\n" + "addu $at, $at, $a0\n" + "addiu $at, $at, 24576\n" + "st.d $w0, -2440($at) # 0xF678\n" + "lui $at, 4661\n" + "addu $at, $at, $a0\n" + "st.d $w0, 120($at)\n" + "st.d $w0, -256($a0)\n" + "st.b $w0, -511($a0)\n" + "addiu $at, $a0, -513\n" + "st.b $w0, 0($at)\n" + "st.h $w0, -1022($a0)\n" + "addiu $at, $a0, -1026\n" + "st.h $w0, 0($at)\n" + "st.w $w0, -2044($a0)\n" + "addiu $at, $a0, -2052\n" + "st.w $w0, 0($at)\n" + "st.d $w0, -4096($a0)\n" + "addiu $at, $a0, -4104\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, -32768\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, -32760\n" + "addiu $at, $at, -4096\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, 32760\n" + "addiu $at, $at, 4096\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, -32760\n" + "addiu $at, $at, -32760\n" + "st.d $w0, -4088($at)\n" + "addiu $at, $a0, 32760\n" + "addiu $at, $at, 32760\n" + "st.d $w0, 4088($at)\n" + "lui $at, 0xABCE\n" + "addu $at, $at, $a0\n" + "addiu $at, $at, -8192 # 0xE000\n" + "st.d $w0, 0xF00($at)\n" + "lui $at, 0x8000\n" + "addu $at, $at, $a0\n" + "addiu $at, $at, -21504 # 0xAC00\n" + "st.b $w0, -51($at) # 0xFFCD\n"; + DriverStr(expected, "StoreQToOffset"); +} + +#undef __ +} // namespace art diff --git a/compiler/utils/mips/assembler_mips32r6_test.cc b/compiler/utils/mips/assembler_mips32r6_test.cc index 30667efa38..6ee2a5cb79 100644 --- a/compiler/utils/mips/assembler_mips32r6_test.cc +++ b/compiler/utils/mips/assembler_mips32r6_test.cc @@ -34,9 +34,14 @@ struct MIPSCpuRegisterCompare { class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, mips::Register, mips::FRegister, - uint32_t> { + uint32_t, + mips::VectorRegister> { public: - typedef AssemblerTest<mips::MipsAssembler, mips::Register, mips::FRegister, uint32_t> Base; + typedef AssemblerTest<mips::MipsAssembler, + mips::Register, + mips::FRegister, + uint32_t, + mips::VectorRegister> Base; AssemblerMIPS32r6Test() : instruction_set_features_(MipsInstructionSetFeatures::FromVariant("mips32r6", nullptr)) { @@ -61,7 +66,7 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, // We use "-modd-spreg" so we can use odd-numbered single precision FPU registers. // We put the code at address 0x1000000 (instead of 0) to avoid overlapping with the // .MIPS.abiflags section (there doesn't seem to be a way to suppress its generation easily). - return " -march=mips32r6 -modd-spreg -Wa,--no-warn" + return " -march=mips32r6 -mmsa -modd-spreg -Wa,--no-warn" " -Wl,-Ttext=0x1000000 -Wl,-e0x1000000 -nostdlib"; } @@ -182,6 +187,39 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, fp_registers_.push_back(new mips::FRegister(mips::F29)); fp_registers_.push_back(new mips::FRegister(mips::F30)); fp_registers_.push_back(new mips::FRegister(mips::F31)); + + vec_registers_.push_back(new mips::VectorRegister(mips::W0)); + vec_registers_.push_back(new mips::VectorRegister(mips::W1)); + vec_registers_.push_back(new mips::VectorRegister(mips::W2)); + vec_registers_.push_back(new mips::VectorRegister(mips::W3)); + vec_registers_.push_back(new mips::VectorRegister(mips::W4)); + vec_registers_.push_back(new mips::VectorRegister(mips::W5)); + vec_registers_.push_back(new mips::VectorRegister(mips::W6)); + vec_registers_.push_back(new mips::VectorRegister(mips::W7)); + vec_registers_.push_back(new mips::VectorRegister(mips::W8)); + vec_registers_.push_back(new mips::VectorRegister(mips::W9)); + vec_registers_.push_back(new mips::VectorRegister(mips::W10)); + vec_registers_.push_back(new mips::VectorRegister(mips::W11)); + vec_registers_.push_back(new mips::VectorRegister(mips::W12)); + vec_registers_.push_back(new mips::VectorRegister(mips::W13)); + vec_registers_.push_back(new mips::VectorRegister(mips::W14)); + vec_registers_.push_back(new mips::VectorRegister(mips::W15)); + vec_registers_.push_back(new mips::VectorRegister(mips::W16)); + vec_registers_.push_back(new mips::VectorRegister(mips::W17)); + vec_registers_.push_back(new mips::VectorRegister(mips::W18)); + vec_registers_.push_back(new mips::VectorRegister(mips::W19)); + vec_registers_.push_back(new mips::VectorRegister(mips::W20)); + vec_registers_.push_back(new mips::VectorRegister(mips::W21)); + vec_registers_.push_back(new mips::VectorRegister(mips::W22)); + vec_registers_.push_back(new mips::VectorRegister(mips::W23)); + vec_registers_.push_back(new mips::VectorRegister(mips::W24)); + vec_registers_.push_back(new mips::VectorRegister(mips::W25)); + vec_registers_.push_back(new mips::VectorRegister(mips::W26)); + vec_registers_.push_back(new mips::VectorRegister(mips::W27)); + vec_registers_.push_back(new mips::VectorRegister(mips::W28)); + vec_registers_.push_back(new mips::VectorRegister(mips::W29)); + vec_registers_.push_back(new mips::VectorRegister(mips::W30)); + vec_registers_.push_back(new mips::VectorRegister(mips::W31)); } } @@ -189,6 +227,7 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, AssemblerTest::TearDown(); STLDeleteElements(®isters_); STLDeleteElements(&fp_registers_); + STLDeleteElements(&vec_registers_); } std::vector<mips::Register*> GetRegisters() OVERRIDE { @@ -199,6 +238,10 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, return fp_registers_; } + std::vector<mips::VectorRegister*> GetVectorRegisters() OVERRIDE { + return vec_registers_; + } + uint32_t CreateImmediate(int64_t imm_value) OVERRIDE { return imm_value; } @@ -250,6 +293,7 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, std::map<mips::Register, std::string, MIPSCpuRegisterCompare> secondary_register_names_; std::vector<mips::FRegister*> fp_registers_; + std::vector<mips::VectorRegister*> vec_registers_; std::unique_ptr<const MipsInstructionSetFeatures> instruction_set_features_; }; @@ -328,13 +372,11 @@ TEST_F(AssemblerMIPS32r6Test, Lsa) { } TEST_F(AssemblerMIPS32r6Test, Seleqz) { - DriverStr(RepeatRRR(&mips::MipsAssembler::Seleqz, "seleqz ${reg1}, ${reg2}, ${reg3}"), - "seleqz"); + DriverStr(RepeatRRR(&mips::MipsAssembler::Seleqz, "seleqz ${reg1}, ${reg2}, ${reg3}"), "seleqz"); } TEST_F(AssemblerMIPS32r6Test, Selnez) { - DriverStr(RepeatRRR(&mips::MipsAssembler::Selnez, "selnez ${reg1}, ${reg2}, ${reg3}"), - "selnez"); + DriverStr(RepeatRRR(&mips::MipsAssembler::Selnez, "selnez ${reg1}, ${reg2}, ${reg3}"), "selnez"); } TEST_F(AssemblerMIPS32r6Test, ClzR6) { @@ -585,6 +627,124 @@ TEST_F(AssemblerMIPS32r6Test, LoadDFromOffset) { DriverStr(expected, "LoadDFromOffset"); } +TEST_F(AssemblerMIPS32r6Test, LoadQFromOffset) { + __ LoadQFromOffset(mips::F0, mips::A0, 0); + __ LoadQFromOffset(mips::F0, mips::A0, 1); + __ LoadQFromOffset(mips::F0, mips::A0, 2); + __ LoadQFromOffset(mips::F0, mips::A0, 4); + __ LoadQFromOffset(mips::F0, mips::A0, 8); + __ LoadQFromOffset(mips::F0, mips::A0, 511); + __ LoadQFromOffset(mips::F0, mips::A0, 512); + __ LoadQFromOffset(mips::F0, mips::A0, 513); + __ LoadQFromOffset(mips::F0, mips::A0, 514); + __ LoadQFromOffset(mips::F0, mips::A0, 516); + __ LoadQFromOffset(mips::F0, mips::A0, 1022); + __ LoadQFromOffset(mips::F0, mips::A0, 1024); + __ LoadQFromOffset(mips::F0, mips::A0, 1025); + __ LoadQFromOffset(mips::F0, mips::A0, 1026); + __ LoadQFromOffset(mips::F0, mips::A0, 1028); + __ LoadQFromOffset(mips::F0, mips::A0, 2044); + __ LoadQFromOffset(mips::F0, mips::A0, 2048); + __ LoadQFromOffset(mips::F0, mips::A0, 2049); + __ LoadQFromOffset(mips::F0, mips::A0, 2050); + __ LoadQFromOffset(mips::F0, mips::A0, 2052); + __ LoadQFromOffset(mips::F0, mips::A0, 4088); + __ LoadQFromOffset(mips::F0, mips::A0, 4096); + __ LoadQFromOffset(mips::F0, mips::A0, 4097); + __ LoadQFromOffset(mips::F0, mips::A0, 4098); + __ LoadQFromOffset(mips::F0, mips::A0, 4100); + __ LoadQFromOffset(mips::F0, mips::A0, 4104); + __ LoadQFromOffset(mips::F0, mips::A0, 0x7FFC); + __ LoadQFromOffset(mips::F0, mips::A0, 0x8000); + __ LoadQFromOffset(mips::F0, mips::A0, 0x10000); + __ LoadQFromOffset(mips::F0, mips::A0, 0x12345678); + __ LoadQFromOffset(mips::F0, mips::A0, 0x12350078); + __ LoadQFromOffset(mips::F0, mips::A0, -256); + __ LoadQFromOffset(mips::F0, mips::A0, -511); + __ LoadQFromOffset(mips::F0, mips::A0, -513); + __ LoadQFromOffset(mips::F0, mips::A0, -1022); + __ LoadQFromOffset(mips::F0, mips::A0, -1026); + __ LoadQFromOffset(mips::F0, mips::A0, -2044); + __ LoadQFromOffset(mips::F0, mips::A0, -2052); + __ LoadQFromOffset(mips::F0, mips::A0, -4096); + __ LoadQFromOffset(mips::F0, mips::A0, -4104); + __ LoadQFromOffset(mips::F0, mips::A0, -32768); + __ LoadQFromOffset(mips::F0, mips::A0, 0xABCDEF00); + __ LoadQFromOffset(mips::F0, mips::A0, 0x7FFFABCD); + + const char* expected = + "ld.d $w0, 0($a0)\n" + "ld.b $w0, 1($a0)\n" + "ld.h $w0, 2($a0)\n" + "ld.w $w0, 4($a0)\n" + "ld.d $w0, 8($a0)\n" + "ld.b $w0, 511($a0)\n" + "ld.d $w0, 512($a0)\n" + "addiu $at, $a0, 513\n" + "ld.b $w0, 0($at)\n" + "ld.h $w0, 514($a0)\n" + "ld.w $w0, 516($a0)\n" + "ld.h $w0, 1022($a0)\n" + "ld.d $w0, 1024($a0)\n" + "addiu $at, $a0, 1025\n" + "ld.b $w0, 0($at)\n" + "addiu $at, $a0, 1026\n" + "ld.h $w0, 0($at)\n" + "ld.w $w0, 1028($a0)\n" + "ld.w $w0, 2044($a0)\n" + "ld.d $w0, 2048($a0)\n" + "addiu $at, $a0, 2049\n" + "ld.b $w0, 0($at)\n" + "addiu $at, $a0, 2050\n" + "ld.h $w0, 0($at)\n" + "addiu $at, $a0, 2052\n" + "ld.w $w0, 0($at)\n" + "ld.d $w0, 4088($a0)\n" + "addiu $at, $a0, 4096\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, 4097\n" + "ld.b $w0, 0($at)\n" + "addiu $at, $a0, 4098\n" + "ld.h $w0, 0($at)\n" + "addiu $at, $a0, 4100\n" + "ld.w $w0, 0($at)\n" + "addiu $at, $a0, 4104\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, 0x7FFC\n" + "ld.w $w0, 0($at)\n" + "addiu $at, $a0, 0x7FF8\n" + "ld.d $w0, 8($at)\n" + "aui $at, $a0, 0x1\n" + "ld.d $w0, 0($at)\n" + "aui $at, $a0, 0x1234\n" + "addiu $at, $at, 0x6000\n" + "ld.d $w0, -2440($at) # 0xF678\n" + "aui $at, $a0, 0x1235\n" + "ld.d $w0, 0x78($at)\n" + "ld.d $w0, -256($a0)\n" + "ld.b $w0, -511($a0)\n" + "addiu $at, $a0, -513\n" + "ld.b $w0, 0($at)\n" + "ld.h $w0, -1022($a0)\n" + "addiu $at, $a0, -1026\n" + "ld.h $w0, 0($at)\n" + "ld.w $w0, -2044($a0)\n" + "addiu $at, $a0, -2052\n" + "ld.w $w0, 0($at)\n" + "ld.d $w0, -4096($a0)\n" + "addiu $at, $a0, -4104\n" + "ld.d $w0, 0($at)\n" + "addiu $at, $a0, -32768\n" + "ld.d $w0, 0($at)\n" + "aui $at, $a0, 0xABCE\n" + "addiu $at, $at, -8192 # 0xE000\n" + "ld.d $w0, 0xF00($at)\n" + "aui $at, $a0, 0x8000\n" + "addiu $at, $at, -21504 # 0xAC00\n" + "ld.b $w0, -51($at) # 0xFFCD\n"; + DriverStr(expected, "LoadQFromOffset"); +} + TEST_F(AssemblerMIPS32r6Test, StoreDToOffset) { __ StoreDToOffset(mips::F0, mips::A0, -0x8000); __ StoreDToOffset(mips::F0, mips::A0, +0); @@ -669,6 +829,124 @@ TEST_F(AssemblerMIPS32r6Test, StoreDToOffset) { DriverStr(expected, "StoreDToOffset"); } +TEST_F(AssemblerMIPS32r6Test, StoreQToOffset) { + __ StoreQToOffset(mips::F0, mips::A0, 0); + __ StoreQToOffset(mips::F0, mips::A0, 1); + __ StoreQToOffset(mips::F0, mips::A0, 2); + __ StoreQToOffset(mips::F0, mips::A0, 4); + __ StoreQToOffset(mips::F0, mips::A0, 8); + __ StoreQToOffset(mips::F0, mips::A0, 511); + __ StoreQToOffset(mips::F0, mips::A0, 512); + __ StoreQToOffset(mips::F0, mips::A0, 513); + __ StoreQToOffset(mips::F0, mips::A0, 514); + __ StoreQToOffset(mips::F0, mips::A0, 516); + __ StoreQToOffset(mips::F0, mips::A0, 1022); + __ StoreQToOffset(mips::F0, mips::A0, 1024); + __ StoreQToOffset(mips::F0, mips::A0, 1025); + __ StoreQToOffset(mips::F0, mips::A0, 1026); + __ StoreQToOffset(mips::F0, mips::A0, 1028); + __ StoreQToOffset(mips::F0, mips::A0, 2044); + __ StoreQToOffset(mips::F0, mips::A0, 2048); + __ StoreQToOffset(mips::F0, mips::A0, 2049); + __ StoreQToOffset(mips::F0, mips::A0, 2050); + __ StoreQToOffset(mips::F0, mips::A0, 2052); + __ StoreQToOffset(mips::F0, mips::A0, 4088); + __ StoreQToOffset(mips::F0, mips::A0, 4096); + __ StoreQToOffset(mips::F0, mips::A0, 4097); + __ StoreQToOffset(mips::F0, mips::A0, 4098); + __ StoreQToOffset(mips::F0, mips::A0, 4100); + __ StoreQToOffset(mips::F0, mips::A0, 4104); + __ StoreQToOffset(mips::F0, mips::A0, 0x7FFC); + __ StoreQToOffset(mips::F0, mips::A0, 0x8000); + __ StoreQToOffset(mips::F0, mips::A0, 0x10000); + __ StoreQToOffset(mips::F0, mips::A0, 0x12345678); + __ StoreQToOffset(mips::F0, mips::A0, 0x12350078); + __ StoreQToOffset(mips::F0, mips::A0, -256); + __ StoreQToOffset(mips::F0, mips::A0, -511); + __ StoreQToOffset(mips::F0, mips::A0, -513); + __ StoreQToOffset(mips::F0, mips::A0, -1022); + __ StoreQToOffset(mips::F0, mips::A0, -1026); + __ StoreQToOffset(mips::F0, mips::A0, -2044); + __ StoreQToOffset(mips::F0, mips::A0, -2052); + __ StoreQToOffset(mips::F0, mips::A0, -4096); + __ StoreQToOffset(mips::F0, mips::A0, -4104); + __ StoreQToOffset(mips::F0, mips::A0, -32768); + __ StoreQToOffset(mips::F0, mips::A0, 0xABCDEF00); + __ StoreQToOffset(mips::F0, mips::A0, 0x7FFFABCD); + + const char* expected = + "st.d $w0, 0($a0)\n" + "st.b $w0, 1($a0)\n" + "st.h $w0, 2($a0)\n" + "st.w $w0, 4($a0)\n" + "st.d $w0, 8($a0)\n" + "st.b $w0, 511($a0)\n" + "st.d $w0, 512($a0)\n" + "addiu $at, $a0, 513\n" + "st.b $w0, 0($at)\n" + "st.h $w0, 514($a0)\n" + "st.w $w0, 516($a0)\n" + "st.h $w0, 1022($a0)\n" + "st.d $w0, 1024($a0)\n" + "addiu $at, $a0, 1025\n" + "st.b $w0, 0($at)\n" + "addiu $at, $a0, 1026\n" + "st.h $w0, 0($at)\n" + "st.w $w0, 1028($a0)\n" + "st.w $w0, 2044($a0)\n" + "st.d $w0, 2048($a0)\n" + "addiu $at, $a0, 2049\n" + "st.b $w0, 0($at)\n" + "addiu $at, $a0, 2050\n" + "st.h $w0, 0($at)\n" + "addiu $at, $a0, 2052\n" + "st.w $w0, 0($at)\n" + "st.d $w0, 4088($a0)\n" + "addiu $at, $a0, 4096\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, 4097\n" + "st.b $w0, 0($at)\n" + "addiu $at, $a0, 4098\n" + "st.h $w0, 0($at)\n" + "addiu $at, $a0, 4100\n" + "st.w $w0, 0($at)\n" + "addiu $at, $a0, 4104\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, 0x7FFC\n" + "st.w $w0, 0($at)\n" + "addiu $at, $a0, 0x7FF8\n" + "st.d $w0, 8($at)\n" + "aui $at, $a0, 0x1\n" + "st.d $w0, 0($at)\n" + "aui $at, $a0, 0x1234\n" + "addiu $at, $at, 0x6000\n" + "st.d $w0, -2440($at) # 0xF678\n" + "aui $at, $a0, 0x1235\n" + "st.d $w0, 0x78($at)\n" + "st.d $w0, -256($a0)\n" + "st.b $w0, -511($a0)\n" + "addiu $at, $a0, -513\n" + "st.b $w0, 0($at)\n" + "st.h $w0, -1022($a0)\n" + "addiu $at, $a0, -1026\n" + "st.h $w0, 0($at)\n" + "st.w $w0, -2044($a0)\n" + "addiu $at, $a0, -2052\n" + "st.w $w0, 0($at)\n" + "st.d $w0, -4096($a0)\n" + "addiu $at, $a0, -4104\n" + "st.d $w0, 0($at)\n" + "addiu $at, $a0, -32768\n" + "st.d $w0, 0($at)\n" + "aui $at, $a0, 0xABCE\n" + "addiu $at, $at, -8192 # 0xE000\n" + "st.d $w0, 0xF00($at)\n" + "aui $at, $a0, 0x8000\n" + "addiu $at, $at, -21504 # 0xAC00\n" + "st.b $w0, -51($at) # 0xFFCD\n"; + DriverStr(expected, "StoreQToOffset"); +} + TEST_F(AssemblerMIPS32r6Test, LoadFarthestNearLabelAddress) { mips::MipsLabel label; __ LoadLabelAddress(mips::V0, mips::ZERO, &label); @@ -914,6 +1192,566 @@ TEST_F(AssemblerMIPS32r6Test, LongBranchReorder) { // AssemblerMIPS32r6Test.Bltu // AssemblerMIPS32r6Test.Bgeu +// MSA instructions. + +TEST_F(AssemblerMIPS32r6Test, AndV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AndV, "and.v ${reg1}, ${reg2}, ${reg3}"), "and.v"); +} + +TEST_F(AssemblerMIPS32r6Test, OrV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::OrV, "or.v ${reg1}, ${reg2}, ${reg3}"), "or.v"); +} + +TEST_F(AssemblerMIPS32r6Test, NorV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::NorV, "nor.v ${reg1}, ${reg2}, ${reg3}"), "nor.v"); +} + +TEST_F(AssemblerMIPS32r6Test, XorV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::XorV, "xor.v ${reg1}, ${reg2}, ${reg3}"), "xor.v"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvB, "addv.b ${reg1}, ${reg2}, ${reg3}"), "addv.b"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvH, "addv.h ${reg1}, ${reg2}, ${reg3}"), "addv.h"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvW, "addv.w ${reg1}, ${reg2}, ${reg3}"), "addv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvD, "addv.d ${reg1}, ${reg2}, ${reg3}"), "addv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvB, "subv.b ${reg1}, ${reg2}, ${reg3}"), "subv.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvH, "subv.h ${reg1}, ${reg2}, ${reg3}"), "subv.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvW, "subv.w ${reg1}, ${reg2}, ${reg3}"), "subv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvD, "subv.d ${reg1}, ${reg2}, ${reg3}"), "subv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvB, "mulv.b ${reg1}, ${reg2}, ${reg3}"), "mulv.b"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvH, "mulv.h ${reg1}, ${reg2}, ${reg3}"), "mulv.h"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvW, "mulv.w ${reg1}, ${reg2}, ${reg3}"), "mulv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvD, "mulv.d ${reg1}, ${reg2}, ${reg3}"), "mulv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sB, "div_s.b ${reg1}, ${reg2}, ${reg3}"), + "div_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sH, "div_s.h ${reg1}, ${reg2}, ${reg3}"), + "div_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sW, "div_s.w ${reg1}, ${reg2}, ${reg3}"), + "div_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sD, "div_s.d ${reg1}, ${reg2}, ${reg3}"), + "div_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uB, "div_u.b ${reg1}, ${reg2}, ${reg3}"), + "div_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uH, "div_u.h ${reg1}, ${reg2}, ${reg3}"), + "div_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uW, "div_u.w ${reg1}, ${reg2}, ${reg3}"), + "div_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uD, "div_u.d ${reg1}, ${reg2}, ${reg3}"), + "div_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sB, "mod_s.b ${reg1}, ${reg2}, ${reg3}"), + "mod_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sH, "mod_s.h ${reg1}, ${reg2}, ${reg3}"), + "mod_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sW, "mod_s.w ${reg1}, ${reg2}, ${reg3}"), + "mod_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sD, "mod_s.d ${reg1}, ${reg2}, ${reg3}"), + "mod_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uB, "mod_u.b ${reg1}, ${reg2}, ${reg3}"), + "mod_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uH, "mod_u.h ${reg1}, ${reg2}, ${reg3}"), + "mod_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uW, "mod_u.w ${reg1}, ${reg2}, ${reg3}"), + "mod_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uD, "mod_u.d ${reg1}, ${reg2}, ${reg3}"), + "mod_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aB, "add_a.b ${reg1}, ${reg2}, ${reg3}"), + "add_a.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aH, "add_a.h ${reg1}, ${reg2}, ${reg3}"), + "add_a.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aW, "add_a.w ${reg1}, ${reg2}, ${reg3}"), + "add_a.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aD, "add_a.d ${reg1}, ${reg2}, ${reg3}"), + "add_a.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sB, "ave_s.b ${reg1}, ${reg2}, ${reg3}"), + "ave_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sH, "ave_s.h ${reg1}, ${reg2}, ${reg3}"), + "ave_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sW, "ave_s.w ${reg1}, ${reg2}, ${reg3}"), + "ave_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sD, "ave_s.d ${reg1}, ${reg2}, ${reg3}"), + "ave_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uB, "ave_u.b ${reg1}, ${reg2}, ${reg3}"), + "ave_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uH, "ave_u.h ${reg1}, ${reg2}, ${reg3}"), + "ave_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uW, "ave_u.w ${reg1}, ${reg2}, ${reg3}"), + "ave_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uD, "ave_u.d ${reg1}, ${reg2}, ${reg3}"), + "ave_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sB, "aver_s.b ${reg1}, ${reg2}, ${reg3}"), + "aver_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sH, "aver_s.h ${reg1}, ${reg2}, ${reg3}"), + "aver_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sW, "aver_s.w ${reg1}, ${reg2}, ${reg3}"), + "aver_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sD, "aver_s.d ${reg1}, ${reg2}, ${reg3}"), + "aver_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uB, "aver_u.b ${reg1}, ${reg2}, ${reg3}"), + "aver_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uH, "aver_u.h ${reg1}, ${reg2}, ${reg3}"), + "aver_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uW, "aver_u.w ${reg1}, ${reg2}, ${reg3}"), + "aver_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uD, "aver_u.d ${reg1}, ${reg2}, ${reg3}"), + "aver_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sB, "max_s.b ${reg1}, ${reg2}, ${reg3}"), + "max_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sH, "max_s.h ${reg1}, ${reg2}, ${reg3}"), + "max_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sW, "max_s.w ${reg1}, ${reg2}, ${reg3}"), + "max_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sD, "max_s.d ${reg1}, ${reg2}, ${reg3}"), + "max_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uB, "max_u.b ${reg1}, ${reg2}, ${reg3}"), + "max_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uH, "max_u.h ${reg1}, ${reg2}, ${reg3}"), + "max_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uW, "max_u.w ${reg1}, ${reg2}, ${reg3}"), + "max_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uD, "max_u.d ${reg1}, ${reg2}, ${reg3}"), + "max_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sB, "min_s.b ${reg1}, ${reg2}, ${reg3}"), + "min_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sH, "min_s.h ${reg1}, ${reg2}, ${reg3}"), + "min_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sW, "min_s.w ${reg1}, ${reg2}, ${reg3}"), + "min_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sD, "min_s.d ${reg1}, ${reg2}, ${reg3}"), + "min_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uB, "min_u.b ${reg1}, ${reg2}, ${reg3}"), + "min_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uH, "min_u.h ${reg1}, ${reg2}, ${reg3}"), + "min_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uW, "min_u.w ${reg1}, ${reg2}, ${reg3}"), + "min_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uD, "min_u.d ${reg1}, ${reg2}, ${reg3}"), + "min_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FaddW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FaddW, "fadd.w ${reg1}, ${reg2}, ${reg3}"), "fadd.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FaddD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FaddD, "fadd.d ${reg1}, ${reg2}, ${reg3}"), "fadd.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FsubW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FsubW, "fsub.w ${reg1}, ${reg2}, ${reg3}"), "fsub.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FsubD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FsubD, "fsub.d ${reg1}, ${reg2}, ${reg3}"), "fsub.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FmulW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmulW, "fmul.w ${reg1}, ${reg2}, ${reg3}"), "fmul.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FmulD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmulD, "fmul.d ${reg1}, ${reg2}, ${reg3}"), "fmul.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FdivW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FdivW, "fdiv.w ${reg1}, ${reg2}, ${reg3}"), "fdiv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FdivD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FdivD, "fdiv.d ${reg1}, ${reg2}, ${reg3}"), "fdiv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FmaxW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmaxW, "fmax.w ${reg1}, ${reg2}, ${reg3}"), "fmax.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FmaxD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmaxD, "fmax.d ${reg1}, ${reg2}, ${reg3}"), "fmax.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FminW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FminW, "fmin.w ${reg1}, ${reg2}, ${reg3}"), "fmin.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FminD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FminD, "fmin.d ${reg1}, ${reg2}, ${reg3}"), "fmin.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ffint_sW) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ffint_sW, "ffint_s.w ${reg1}, ${reg2}"), "ffint_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ffint_sD) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ffint_sD, "ffint_s.d ${reg1}, ${reg2}"), "ffint_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ftint_sW) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ftint_sW, "ftint_s.w ${reg1}, ${reg2}"), "ftint_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ftint_sD) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ftint_sD, "ftint_s.d ${reg1}, ${reg2}"), "ftint_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SllB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllB, "sll.b ${reg1}, ${reg2}, ${reg3}"), "sll.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SllH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllH, "sll.h ${reg1}, ${reg2}, ${reg3}"), "sll.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SllW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllW, "sll.w ${reg1}, ${reg2}, ${reg3}"), "sll.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SllD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllD, "sll.d ${reg1}, ${reg2}, ${reg3}"), "sll.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SraB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraB, "sra.b ${reg1}, ${reg2}, ${reg3}"), "sra.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SraH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraH, "sra.h ${reg1}, ${reg2}, ${reg3}"), "sra.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SraW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraW, "sra.w ${reg1}, ${reg2}, ${reg3}"), "sra.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SraD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraD, "sra.d ${reg1}, ${reg2}, ${reg3}"), "sra.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlB, "srl.b ${reg1}, ${reg2}, ${reg3}"), "srl.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlH, "srl.h ${reg1}, ${reg2}, ${reg3}"), "srl.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlW, "srl.w ${reg1}, ${reg2}, ${reg3}"), "srl.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlD, "srl.d ${reg1}, ${reg2}, ${reg3}"), "srl.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliB) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliB, 3, "slli.b ${reg1}, ${reg2}, {imm}"), "slli.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliH) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliH, 4, "slli.h ${reg1}, ${reg2}, {imm}"), "slli.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliW) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliW, 5, "slli.w ${reg1}, ${reg2}, {imm}"), "slli.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliD) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliD, 6, "slli.d ${reg1}, ${reg2}, {imm}"), "slli.d"); +} + +TEST_F(AssemblerMIPS32r6Test, MoveV) { + DriverStr(RepeatVV(&mips::MipsAssembler::MoveV, "move.v ${reg1}, ${reg2}"), "move.v"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiB) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiB, 4, "splati.b ${reg1}, ${reg2}[{imm}]"), + "splati.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiH) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiH, 3, "splati.h ${reg1}, ${reg2}[{imm}]"), + "splati.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiW) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiW, 2, "splati.w ${reg1}, ${reg2}[{imm}]"), + "splati.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiD) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiD, 1, "splati.d ${reg1}, ${reg2}[{imm}]"), + "splati.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FillB) { + DriverStr(RepeatVR(&mips::MipsAssembler::FillB, "fill.b ${reg1}, ${reg2}"), "fill.b"); +} + +TEST_F(AssemblerMIPS32r6Test, FillH) { + DriverStr(RepeatVR(&mips::MipsAssembler::FillH, "fill.h ${reg1}, ${reg2}"), "fill.h"); +} + +TEST_F(AssemblerMIPS32r6Test, FillW) { + DriverStr(RepeatVR(&mips::MipsAssembler::FillW, "fill.w ${reg1}, ${reg2}"), "fill.w"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiB) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiB, -8, "ldi.b ${reg}, {imm}"), "ldi.b"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiH) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiH, -10, "ldi.h ${reg}, {imm}"), "ldi.h"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiW) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiW, -10, "ldi.w ${reg}, {imm}"), "ldi.w"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiD) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiD, -10, "ldi.d ${reg}, {imm}"), "ldi.d"); +} + +TEST_F(AssemblerMIPS32r6Test, LdB) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdB, -10, "ld.b ${reg1}, {imm}(${reg2})"), "ld.b"); +} + +TEST_F(AssemblerMIPS32r6Test, LdH) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdH, -10, "ld.h ${reg1}, {imm}(${reg2})", 0, 2), + "ld.h"); +} + +TEST_F(AssemblerMIPS32r6Test, LdW) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdW, -10, "ld.w ${reg1}, {imm}(${reg2})", 0, 4), + "ld.w"); +} + +TEST_F(AssemblerMIPS32r6Test, LdD) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdD, -10, "ld.d ${reg1}, {imm}(${reg2})", 0, 8), + "ld.d"); +} + +TEST_F(AssemblerMIPS32r6Test, StB) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StB, -10, "st.b ${reg1}, {imm}(${reg2})"), "st.b"); +} + +TEST_F(AssemblerMIPS32r6Test, StH) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StH, -10, "st.h ${reg1}, {imm}(${reg2})", 0, 2), + "st.h"); +} + +TEST_F(AssemblerMIPS32r6Test, StW) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StW, -10, "st.w ${reg1}, {imm}(${reg2})", 0, 4), + "st.w"); +} + +TEST_F(AssemblerMIPS32r6Test, StD) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StD, -10, "st.d ${reg1}, {imm}(${reg2})", 0, 8), + "st.d"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrB, "ilvr.b ${reg1}, ${reg2}, ${reg3}"), "ilvr.b"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrH, "ilvr.h ${reg1}, ${reg2}, ${reg3}"), "ilvr.h"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrW, "ilvr.w ${reg1}, ${reg2}, ${reg3}"), "ilvr.w"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrD, "ilvr.d ${reg1}, ${reg2}, ${reg3}"), "ilvr.d"); +} + #undef __ } // namespace art diff --git a/compiler/utils/mips/assembler_mips_test.cc b/compiler/utils/mips/assembler_mips_test.cc index c24e1b16fb..09175309f9 100644 --- a/compiler/utils/mips/assembler_mips_test.cc +++ b/compiler/utils/mips/assembler_mips_test.cc @@ -2851,7 +2851,7 @@ TEST_F(AssemblerMIPSTest, LongBranchReorder) { // Account for 5 extra instructions: ori, addu, lw, jalr, addiu. uint32_t offset_forward = (kAdduCount1 + 5) * sizeof(uint32_t); // Account for 5 extra instructions: subu, addiu, sw, nal, lui. - uint32_t offset_back = -(kAdduCount1 + 5) * sizeof(uint32_t); + uint32_t offset_back = static_cast<uint32_t>(-(kAdduCount1 + 5) * sizeof(uint32_t)); std::ostringstream oss; oss << diff --git a/compiler/utils/mips/constants_mips.h b/compiler/utils/mips/constants_mips.h index 44ed5cc124..b4dfdbd8d3 100644 --- a/compiler/utils/mips/constants_mips.h +++ b/compiler/utils/mips/constants_mips.h @@ -75,8 +75,37 @@ enum InstructionFields { kFdShift = 6, kFdBits = 5, + kMsaOperationShift = 23, + kMsaELMOperationShift = 22, + kMsa2ROperationShift = 18, + kMsa2RFOperationShift = 17, + kDfShift = 21, + kDfMShift = 16, + kDf2RShift = 16, + kDfNShift = 16, + kWtShift = 16, + kWtBits = 5, + kWsShift = 11, + kWsBits = 5, + kWdShift = 6, + kWdBits = 5, + kS10Shift = 16, + kI10Shift = 11, + kS10MinorShift = 2, + kBranchOffsetMask = 0x0000ffff, kJumpOffsetMask = 0x03ffffff, + + kMsaMajorOpcode = 0x1e, + kMsaDfMByteMask = 0x70, + kMsaDfMHalfwordMask = 0x60, + kMsaDfMWordMask = 0x40, + kMsaDfMDoublewordMask = 0x00, + kMsaDfNByteMask = 0x00, + kMsaDfNHalfwordMask = 0x20, + kMsaDfNWordMask = 0x30, + kMsaDfNDoublewordMask = 0x38, + kMsaS10Mask = 0x3ff, }; enum ScaleFactor { diff --git a/compiler/utils/mips64/assembler_mips64.cc b/compiler/utils/mips64/assembler_mips64.cc index b8b800abe3..24900a7f10 100644 --- a/compiler/utils/mips64/assembler_mips64.cc +++ b/compiler/utils/mips64/assembler_mips64.cc @@ -1456,6 +1456,86 @@ void Mips64Assembler::Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegist EmitMsa3R(0x7, 0x3, wt, ws, wd, 0x10); } +void Mips64Assembler::Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x3, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x3, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x3, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x3, wt, ws, wd, 0xe); +} + void Mips64Assembler::FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { CHECK(HasMsa()); EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x1b); @@ -1496,6 +1576,26 @@ void Mips64Assembler::FdivD(VectorRegister wd, VectorRegister ws, VectorRegister EmitMsa3R(0x1, 0x3, wt, ws, wd, 0x1b); } +void Mips64Assembler::FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x1b); +} + +void Mips64Assembler::FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x1b); +} + +void Mips64Assembler::FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x1b); +} + +void Mips64Assembler::FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x1b); +} + void Mips64Assembler::Ffint_sW(VectorRegister wd, VectorRegister ws) { CHECK(HasMsa()); EmitMsa2RF(0x19e, 0x0, ws, wd, 0x1e); diff --git a/compiler/utils/mips64/assembler_mips64.h b/compiler/utils/mips64/assembler_mips64.h index 9b4064543f..773db9b208 100644 --- a/compiler/utils/mips64/assembler_mips64.h +++ b/compiler/utils/mips64/assembler_mips64.h @@ -25,6 +25,7 @@ #include "base/arena_containers.h" #include "base/enums.h" #include "base/macros.h" +#include "base/stl_util_identity.h" #include "constants_mips64.h" #include "globals.h" #include "managed_register_mips64.h" @@ -704,6 +705,22 @@ class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<Pointer void Aver_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Aver_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FaddD(VectorRegister wd, VectorRegister ws, VectorRegister wt); @@ -713,6 +730,10 @@ class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<Pointer void FmulD(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FdivW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FdivD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Ffint_sW(VectorRegister wd, VectorRegister ws); void Ffint_sD(VectorRegister wd, VectorRegister ws); diff --git a/compiler/utils/mips64/assembler_mips64_test.cc b/compiler/utils/mips64/assembler_mips64_test.cc index fbebe0ce15..bdf9598ee7 100644 --- a/compiler/utils/mips64/assembler_mips64_test.cc +++ b/compiler/utils/mips64/assembler_mips64_test.cc @@ -2998,6 +2998,86 @@ TEST_F(AssemblerMIPS64Test, Aver_uD) { "aver_u.d"); } +TEST_F(AssemblerMIPS64Test, Max_sB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sB, "max_s.b ${reg1}, ${reg2}, ${reg3}"), + "max_s.b"); +} + +TEST_F(AssemblerMIPS64Test, Max_sH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sH, "max_s.h ${reg1}, ${reg2}, ${reg3}"), + "max_s.h"); +} + +TEST_F(AssemblerMIPS64Test, Max_sW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sW, "max_s.w ${reg1}, ${reg2}, ${reg3}"), + "max_s.w"); +} + +TEST_F(AssemblerMIPS64Test, Max_sD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sD, "max_s.d ${reg1}, ${reg2}, ${reg3}"), + "max_s.d"); +} + +TEST_F(AssemblerMIPS64Test, Max_uB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uB, "max_u.b ${reg1}, ${reg2}, ${reg3}"), + "max_u.b"); +} + +TEST_F(AssemblerMIPS64Test, Max_uH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uH, "max_u.h ${reg1}, ${reg2}, ${reg3}"), + "max_u.h"); +} + +TEST_F(AssemblerMIPS64Test, Max_uW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uW, "max_u.w ${reg1}, ${reg2}, ${reg3}"), + "max_u.w"); +} + +TEST_F(AssemblerMIPS64Test, Max_uD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uD, "max_u.d ${reg1}, ${reg2}, ${reg3}"), + "max_u.d"); +} + +TEST_F(AssemblerMIPS64Test, Min_sB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sB, "min_s.b ${reg1}, ${reg2}, ${reg3}"), + "min_s.b"); +} + +TEST_F(AssemblerMIPS64Test, Min_sH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sH, "min_s.h ${reg1}, ${reg2}, ${reg3}"), + "min_s.h"); +} + +TEST_F(AssemblerMIPS64Test, Min_sW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sW, "min_s.w ${reg1}, ${reg2}, ${reg3}"), + "min_s.w"); +} + +TEST_F(AssemblerMIPS64Test, Min_sD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sD, "min_s.d ${reg1}, ${reg2}, ${reg3}"), + "min_s.d"); +} + +TEST_F(AssemblerMIPS64Test, Min_uB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uB, "min_u.b ${reg1}, ${reg2}, ${reg3}"), + "min_u.b"); +} + +TEST_F(AssemblerMIPS64Test, Min_uH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uH, "min_u.h ${reg1}, ${reg2}, ${reg3}"), + "min_u.h"); +} + +TEST_F(AssemblerMIPS64Test, Min_uW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uW, "min_u.w ${reg1}, ${reg2}, ${reg3}"), + "min_u.w"); +} + +TEST_F(AssemblerMIPS64Test, Min_uD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uD, "min_u.d ${reg1}, ${reg2}, ${reg3}"), + "min_u.d"); +} + TEST_F(AssemblerMIPS64Test, FaddW) { DriverStr(RepeatVVV(&mips64::Mips64Assembler::FaddW, "fadd.w ${reg1}, ${reg2}, ${reg3}"), "fadd.w"); @@ -3038,6 +3118,26 @@ TEST_F(AssemblerMIPS64Test, FdivD) { "fdiv.d"); } +TEST_F(AssemblerMIPS64Test, FmaxW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FmaxW, "fmax.w ${reg1}, ${reg2}, ${reg3}"), + "fmax.w"); +} + +TEST_F(AssemblerMIPS64Test, FmaxD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FmaxD, "fmax.d ${reg1}, ${reg2}, ${reg3}"), + "fmax.d"); +} + +TEST_F(AssemblerMIPS64Test, FminW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FminW, "fmin.w ${reg1}, ${reg2}, ${reg3}"), + "fmin.w"); +} + +TEST_F(AssemblerMIPS64Test, FminD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FminD, "fmin.d ${reg1}, ${reg2}, ${reg3}"), + "fmin.d"); +} + TEST_F(AssemblerMIPS64Test, Ffint_sW) { DriverStr(RepeatVV(&mips64::Mips64Assembler::Ffint_sW, "ffint_s.w ${reg1}, ${reg2}"), "ffint_s.w"); diff --git a/compiler/utils/swap_space.cc b/compiler/utils/swap_space.cc index 4f6c915142..621a652f0a 100644 --- a/compiler/utils/swap_space.cc +++ b/compiler/utils/swap_space.cc @@ -20,6 +20,7 @@ #include <numeric> #include <sys/mman.h> +#include "base/bit_utils.h" #include "base/logging.h" #include "base/macros.h" #include "base/mutex.h" diff --git a/compiler/verifier_deps_test.cc b/compiler/verifier_deps_test.cc index dd09fed06e..65389252e2 100644 --- a/compiler/verifier_deps_test.cc +++ b/compiler/verifier_deps_test.cc @@ -33,7 +33,7 @@ #include "runtime.h" #include "scoped_thread_state_change-inl.h" #include "thread.h" -#include "utils/atomic_method_ref_map-inl.h" +#include "utils/atomic_dex_ref_map-inl.h" #include "verifier/method_verifier-inl.h" namespace art { @@ -87,19 +87,19 @@ class VerifierDepsTest : public CommonCompilerTest { TimingLogger timings("Verify", false, false); // The compiler driver handles the verifier deps in the callbacks, so // remove what this class did for unit testing. - verifier_deps_.reset(nullptr); + if (deps == nullptr) { + // Create some verifier deps by default if they are not already specified. + deps = new verifier::VerifierDeps(dex_files_); + verifier_deps_.reset(deps); + } callbacks_->SetVerifierDeps(deps); compiler_driver_->Verify(class_loader_, dex_files_, &timings); - // The compiler driver may have updated the VerifierDeps in the callback object. - if (callbacks_->GetVerifierDeps() != deps) { - verifier_deps_.reset(callbacks_->GetVerifierDeps()); - } callbacks_->SetVerifierDeps(nullptr); // Clear entries in the verification results to avoid hitting a DCHECK that // we always succeed inserting a new entry after verifying. - AtomicMethodRefMap<const VerifiedMethod*>* map = + AtomicDexRefMap<const VerifiedMethod*>* map = &compiler_driver_->GetVerificationResults()->atomic_verified_methods_; - map->Visit([](const MethodReference& ref ATTRIBUTE_UNUSED, const VerifiedMethod* method) { + map->Visit([](const DexFileReference& ref ATTRIBUTE_UNUSED, const VerifiedMethod* method) { delete method; }); map->ClearEntries(); @@ -128,6 +128,7 @@ class VerifierDepsTest : public CommonCompilerTest { for (const DexFile* dex_file : dex_files_) { compiler_driver_->GetVerificationResults()->AddDexFile(dex_file); } + compiler_driver_->SetDexFilesForOatFile(dex_files_); } void LoadDexFile(ScopedObjectAccess* soa) REQUIRES_SHARED(Locks::mutator_lock_) { @@ -151,19 +152,18 @@ class VerifierDepsTest : public CommonCompilerTest { CHECK(class_data != nullptr); ClassDataItemIterator it(*primary_dex_file_, class_data); - while (it.HasNextStaticField() || it.HasNextInstanceField()) { - it.Next(); - } + it.SkipAllFields(); ArtMethod* method = nullptr; while (it.HasNextDirectMethod()) { - ArtMethod* resolved_method = class_linker_->ResolveMethod<ClassLinker::kNoICCECheckForCache>( - *primary_dex_file_, - it.GetMemberIndex(), - dex_cache_handle, - class_loader_handle, - nullptr, - it.GetMethodInvokeType(*class_def)); + ArtMethod* resolved_method = + class_linker_->ResolveMethod<ClassLinker::ResolveMode::kNoChecks>( + *primary_dex_file_, + it.GetMemberIndex(), + dex_cache_handle, + class_loader_handle, + nullptr, + it.GetMethodInvokeType(*class_def)); CHECK(resolved_method != nullptr); if (method_name == resolved_method->GetName()) { method = resolved_method; @@ -229,8 +229,7 @@ class VerifierDepsTest : public CommonCompilerTest { hs.NewHandle(soa.Decode<mirror::ClassLoader>(class_loader_))); MutableHandle<mirror::Class> cls(hs.NewHandle<mirror::Class>(nullptr)); for (const DexFile* dex_file : dex_files_) { - const std::vector<dex::TypeIndex>& unverified_classes = deps.GetUnverifiedClasses(*dex_file); - std::set<dex::TypeIndex> set(unverified_classes.begin(), unverified_classes.end()); + const std::set<dex::TypeIndex>& unverified_classes = deps.GetUnverifiedClasses(*dex_file); for (uint32_t i = 0; i < dex_file->NumClassDefs(); ++i) { const DexFile::ClassDef& class_def = dex_file->GetClassDef(i); const char* descriptor = dex_file->GetClassDescriptor(class_def); @@ -238,7 +237,7 @@ class VerifierDepsTest : public CommonCompilerTest { if (cls == nullptr) { CHECK(soa.Self()->IsExceptionPending()); soa.Self()->ClearException(); - } else if (set.find(class_def.class_idx_) == set.end()) { + } else if (unverified_classes.find(class_def.class_idx_) == unverified_classes.end()) { ASSERT_EQ(cls->GetStatus(), mirror::Class::kStatusVerified); } else { ASSERT_LT(cls->GetStatus(), mirror::Class::kStatusVerified); @@ -371,18 +370,14 @@ class VerifierDepsTest : public CommonCompilerTest { // Iterates over all method resolution records, finds an entry which matches // the given field kind+class+name+signature and tests its properties. - bool HasMethod(const std::string& expected_kind, - const std::string& expected_klass, + bool HasMethod(const std::string& expected_klass, const std::string& expected_name, const std::string& expected_signature, bool expected_resolved, const std::string& expected_access_flags = "", const std::string& expected_decl_klass = "") { for (auto& dex_dep : verifier_deps_->dex_deps_) { - auto& storage = (expected_kind == "direct") ? dex_dep.second->direct_methods_ - : (expected_kind == "virtual") ? dex_dep.second->virtual_methods_ - : dex_dep.second->interface_methods_; - for (auto& entry : storage) { + for (const VerifierDeps::MethodResolution& entry : dex_dep.second->methods_) { if (expected_resolved != entry.IsResolved()) { continue; } @@ -443,9 +438,7 @@ class VerifierDepsTest : public CommonCompilerTest { has_assignability |= !entry.second->unassignable_types_.empty(); has_classes |= !entry.second->classes_.empty(); has_fields |= !entry.second->fields_.empty(); - has_methods |= !entry.second->direct_methods_.empty(); - has_methods |= !entry.second->virtual_methods_.empty(); - has_methods |= !entry.second->interface_methods_.empty(); + has_methods |= !entry.second->methods_.empty(); has_unverified_classes |= !entry.second->unverified_classes_.empty(); } @@ -457,18 +450,6 @@ class VerifierDepsTest : public CommonCompilerTest { has_unverified_classes; } - static std::set<VerifierDeps::MethodResolution>* GetMethods( - VerifierDeps::DexFileDeps* deps, MethodResolutionKind resolution_kind) { - if (resolution_kind == kDirectMethodResolution) { - return &deps->direct_methods_; - } else if (resolution_kind == kVirtualMethodResolution) { - return &deps->virtual_methods_; - } else { - DCHECK_EQ(resolution_kind, kInterfaceMethodResolution); - return &deps->interface_methods_; - } - } - std::unique_ptr<verifier::VerifierDeps> verifier_deps_; std::vector<const DexFile*> dex_files_; const DexFile* primary_dex_file_; @@ -606,11 +587,10 @@ TEST_F(VerifierDepsTest, InvokeArgumentType) { ASSERT_TRUE(VerifyMethod("InvokeArgumentType")); ASSERT_TRUE(HasClass("Ljava/text/SimpleDateFormat;", true, "public")); ASSERT_TRUE(HasClass("Ljava/util/SimpleTimeZone;", true, "public")); - ASSERT_TRUE(HasMethod("virtual", - "Ljava/text/SimpleDateFormat;", + ASSERT_TRUE(HasMethod("Ljava/text/SimpleDateFormat;", "setTimeZone", "(Ljava/util/TimeZone;)V", - true, + /* expect_resolved */ true, "public", "Ljava/text/DateFormat;")); ASSERT_TRUE(HasAssignable("Ljava/util/TimeZone;", "Ljava/util/SimpleTimeZone;", true)); @@ -842,11 +822,10 @@ TEST_F(VerifierDepsTest, InstanceField_Unresolved_ReferrerInDex) { TEST_F(VerifierDepsTest, InvokeStatic_Resolved_DeclaredInReferenced) { ASSERT_TRUE(VerifyMethod("InvokeStatic_Resolved_DeclaredInReferenced")); ASSERT_TRUE(HasClass("Ljava/net/Socket;", true, "public")); - ASSERT_TRUE(HasMethod("direct", - "Ljava/net/Socket;", + ASSERT_TRUE(HasMethod("Ljava/net/Socket;", "setSocketImplFactory", "(Ljava/net/SocketImplFactory;)V", - true, + /* expect_resolved */ true, "public static", "Ljava/net/Socket;")); } @@ -854,22 +833,20 @@ TEST_F(VerifierDepsTest, InvokeStatic_Resolved_DeclaredInReferenced) { TEST_F(VerifierDepsTest, InvokeStatic_Resolved_DeclaredInSuperclass1) { ASSERT_TRUE(VerifyMethod("InvokeStatic_Resolved_DeclaredInSuperclass1")); ASSERT_TRUE(HasClass("Ljavax/net/ssl/SSLSocket;", true, "public")); - ASSERT_TRUE(HasMethod("direct", - "Ljavax/net/ssl/SSLSocket;", + ASSERT_TRUE(HasMethod("Ljavax/net/ssl/SSLSocket;", "setSocketImplFactory", "(Ljava/net/SocketImplFactory;)V", - true, + /* expect_resolved */ true, "public static", "Ljava/net/Socket;")); } TEST_F(VerifierDepsTest, InvokeStatic_Resolved_DeclaredInSuperclass2) { ASSERT_TRUE(VerifyMethod("InvokeStatic_Resolved_DeclaredInSuperclass2")); - ASSERT_TRUE(HasMethod("direct", - "LMySSLSocket;", + ASSERT_TRUE(HasMethod("LMySSLSocket;", "setSocketImplFactory", "(Ljava/net/SocketImplFactory;)V", - true, + /* expect_resolved */ true, "public static", "Ljava/net/Socket;")); } @@ -877,11 +854,10 @@ TEST_F(VerifierDepsTest, InvokeStatic_Resolved_DeclaredInSuperclass2) { TEST_F(VerifierDepsTest, InvokeStatic_DeclaredInInterface1) { ASSERT_TRUE(VerifyMethod("InvokeStatic_DeclaredInInterface1")); ASSERT_TRUE(HasClass("Ljava/util/Map$Entry;", true, "public interface")); - ASSERT_TRUE(HasMethod("direct", - "Ljava/util/Map$Entry;", + ASSERT_TRUE(HasMethod("Ljava/util/Map$Entry;", "comparingByKey", "()Ljava/util/Comparator;", - true, + /* expect_resolved */ true, "public static", "Ljava/util/Map$Entry;")); } @@ -889,68 +865,85 @@ TEST_F(VerifierDepsTest, InvokeStatic_DeclaredInInterface1) { TEST_F(VerifierDepsTest, InvokeStatic_DeclaredInInterface2) { ASSERT_FALSE(VerifyMethod("InvokeStatic_DeclaredInInterface2")); ASSERT_TRUE(HasClass("Ljava/util/AbstractMap$SimpleEntry;", true, "public")); - ASSERT_TRUE(HasMethod("direct", - "Ljava/util/AbstractMap$SimpleEntry;", + ASSERT_TRUE(HasMethod("Ljava/util/AbstractMap$SimpleEntry;", "comparingByKey", "()Ljava/util/Comparator;", - false)); + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeStatic_Unresolved1) { ASSERT_FALSE(VerifyMethod("InvokeStatic_Unresolved1")); ASSERT_TRUE(HasClass("Ljavax/net/ssl/SSLSocket;", true, "public")); - ASSERT_TRUE(HasMethod("direct", "Ljavax/net/ssl/SSLSocket;", "x", "()V", false)); + ASSERT_TRUE(HasMethod("Ljavax/net/ssl/SSLSocket;", + "x", + "()V", + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeStatic_Unresolved2) { ASSERT_FALSE(VerifyMethod("InvokeStatic_Unresolved2")); - ASSERT_TRUE(HasMethod("direct", "LMySSLSocket;", "x", "()V", false)); + ASSERT_TRUE(HasMethod("LMySSLSocket;", + "x", + "()V", + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeDirect_Resolved_DeclaredInReferenced) { ASSERT_TRUE(VerifyMethod("InvokeDirect_Resolved_DeclaredInReferenced")); ASSERT_TRUE(HasClass("Ljava/net/Socket;", true, "public")); - ASSERT_TRUE(HasMethod( - "direct", "Ljava/net/Socket;", "<init>", "()V", true, "public", "Ljava/net/Socket;")); + ASSERT_TRUE(HasMethod("Ljava/net/Socket;", + "<init>", + "()V", + /* expect_resolved */ true, + "public", + "Ljava/net/Socket;")); } TEST_F(VerifierDepsTest, InvokeDirect_Resolved_DeclaredInSuperclass1) { ASSERT_FALSE(VerifyMethod("InvokeDirect_Resolved_DeclaredInSuperclass1")); ASSERT_TRUE(HasClass("Ljavax/net/ssl/SSLSocket;", true, "public")); - ASSERT_TRUE(HasMethod("direct", - "Ljavax/net/ssl/SSLSocket;", + ASSERT_TRUE(HasMethod("Ljavax/net/ssl/SSLSocket;", "checkOldImpl", "()V", - true, + /* expect_resolved */ true, "private", "Ljava/net/Socket;")); } TEST_F(VerifierDepsTest, InvokeDirect_Resolved_DeclaredInSuperclass2) { ASSERT_FALSE(VerifyMethod("InvokeDirect_Resolved_DeclaredInSuperclass2")); - ASSERT_TRUE(HasMethod( - "direct", "LMySSLSocket;", "checkOldImpl", "()V", true, "private", "Ljava/net/Socket;")); + ASSERT_TRUE(HasMethod("LMySSLSocket;", + "checkOldImpl", + "()V", + /* expect_resolved */ true, + "private", + "Ljava/net/Socket;")); } TEST_F(VerifierDepsTest, InvokeDirect_Unresolved1) { ASSERT_FALSE(VerifyMethod("InvokeDirect_Unresolved1")); ASSERT_TRUE(HasClass("Ljavax/net/ssl/SSLSocket;", true, "public")); - ASSERT_TRUE(HasMethod("direct", "Ljavax/net/ssl/SSLSocket;", "x", "()V", false)); + ASSERT_TRUE(HasMethod("Ljavax/net/ssl/SSLSocket;", + "x", + "()V", + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeDirect_Unresolved2) { ASSERT_FALSE(VerifyMethod("InvokeDirect_Unresolved2")); - ASSERT_TRUE(HasMethod("direct", "LMySSLSocket;", "x", "()V", false)); + ASSERT_TRUE(HasMethod("LMySSLSocket;", + "x", + "()V", + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeVirtual_Resolved_DeclaredInReferenced) { ASSERT_TRUE(VerifyMethod("InvokeVirtual_Resolved_DeclaredInReferenced")); ASSERT_TRUE(HasClass("Ljava/lang/Throwable;", true, "public")); - ASSERT_TRUE(HasMethod("virtual", - "Ljava/lang/Throwable;", + ASSERT_TRUE(HasMethod("Ljava/lang/Throwable;", "getMessage", "()Ljava/lang/String;", - true, + /* expect_resolved */ true, "public", "Ljava/lang/Throwable;")); // Type dependency on `this` argument. @@ -960,11 +953,10 @@ TEST_F(VerifierDepsTest, InvokeVirtual_Resolved_DeclaredInReferenced) { TEST_F(VerifierDepsTest, InvokeVirtual_Resolved_DeclaredInSuperclass1) { ASSERT_TRUE(VerifyMethod("InvokeVirtual_Resolved_DeclaredInSuperclass1")); ASSERT_TRUE(HasClass("Ljava/io/InterruptedIOException;", true, "public")); - ASSERT_TRUE(HasMethod("virtual", - "Ljava/io/InterruptedIOException;", + ASSERT_TRUE(HasMethod("Ljava/io/InterruptedIOException;", "getMessage", "()Ljava/lang/String;", - true, + /* expect_resolved */ true, "public", "Ljava/lang/Throwable;")); // Type dependency on `this` argument. @@ -973,22 +965,20 @@ TEST_F(VerifierDepsTest, InvokeVirtual_Resolved_DeclaredInSuperclass1) { TEST_F(VerifierDepsTest, InvokeVirtual_Resolved_DeclaredInSuperclass2) { ASSERT_TRUE(VerifyMethod("InvokeVirtual_Resolved_DeclaredInSuperclass2")); - ASSERT_TRUE(HasMethod("virtual", - "LMySocketTimeoutException;", + ASSERT_TRUE(HasMethod("LMySocketTimeoutException;", "getMessage", "()Ljava/lang/String;", - true, + /* expect_resolved */ true, "public", "Ljava/lang/Throwable;")); } TEST_F(VerifierDepsTest, InvokeVirtual_Resolved_DeclaredInSuperinterface) { ASSERT_TRUE(VerifyMethod("InvokeVirtual_Resolved_DeclaredInSuperinterface")); - ASSERT_TRUE(HasMethod("virtual", - "LMyThreadSet;", + ASSERT_TRUE(HasMethod("LMyThreadSet;", "size", "()I", - true, + /* expect_resolved */ true, "public", "Ljava/util/Set;")); } @@ -996,61 +986,59 @@ TEST_F(VerifierDepsTest, InvokeVirtual_Resolved_DeclaredInSuperinterface) { TEST_F(VerifierDepsTest, InvokeVirtual_Unresolved1) { ASSERT_FALSE(VerifyMethod("InvokeVirtual_Unresolved1")); ASSERT_TRUE(HasClass("Ljava/io/InterruptedIOException;", true, "public")); - ASSERT_TRUE(HasMethod("virtual", "Ljava/io/InterruptedIOException;", "x", "()V", false)); + ASSERT_TRUE(HasMethod("Ljava/io/InterruptedIOException;", + "x", + "()V", + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeVirtual_Unresolved2) { ASSERT_FALSE(VerifyMethod("InvokeVirtual_Unresolved2")); - ASSERT_TRUE(HasMethod("virtual", "LMySocketTimeoutException;", "x", "()V", false)); -} - -TEST_F(VerifierDepsTest, InvokeVirtual_ActuallyDirect) { - ASSERT_FALSE(VerifyMethod("InvokeVirtual_ActuallyDirect")); - ASSERT_TRUE(HasMethod("virtual", "LMyThread;", "activeCount", "()I", false)); - ASSERT_TRUE(HasMethod("direct", - "LMyThread;", - "activeCount", - "()I", - true, - "public static", - "Ljava/lang/Thread;")); + ASSERT_TRUE(HasMethod("LMySocketTimeoutException;", + "x", + "()V", + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeInterface_Resolved_DeclaredInReferenced) { ASSERT_TRUE(VerifyMethod("InvokeInterface_Resolved_DeclaredInReferenced")); ASSERT_TRUE(HasClass("Ljava/lang/Runnable;", true, "public interface")); - ASSERT_TRUE(HasMethod("interface", - "Ljava/lang/Runnable;", + ASSERT_TRUE(HasMethod("Ljava/lang/Runnable;", "run", "()V", - true, + /* expect_resolved */ true, "public", "Ljava/lang/Runnable;")); } TEST_F(VerifierDepsTest, InvokeInterface_Resolved_DeclaredInSuperclass) { ASSERT_FALSE(VerifyMethod("InvokeInterface_Resolved_DeclaredInSuperclass")); - ASSERT_TRUE(HasMethod("interface", "LMyThread;", "join", "()V", false)); + // TODO: Maybe we should not record dependency if the invoke type does not match the lookup type. + ASSERT_TRUE(HasMethod("LMyThread;", + "join", + "()V", + /* expect_resolved */ true, + "public", + "Ljava/lang/Thread;")); } TEST_F(VerifierDepsTest, InvokeInterface_Resolved_DeclaredInSuperinterface1) { ASSERT_FALSE(VerifyMethod("InvokeInterface_Resolved_DeclaredInSuperinterface1")); - ASSERT_TRUE(HasMethod("interface", - "LMyThreadSet;", + // TODO: Maybe we should not record dependency if the invoke type does not match the lookup type. + ASSERT_TRUE(HasMethod("LMyThreadSet;", "run", "()V", - true, + /* expect_resolved */ true, "public", - "Ljava/lang/Runnable;")); + "Ljava/lang/Thread;")); } TEST_F(VerifierDepsTest, InvokeInterface_Resolved_DeclaredInSuperinterface2) { ASSERT_FALSE(VerifyMethod("InvokeInterface_Resolved_DeclaredInSuperinterface2")); - ASSERT_TRUE(HasMethod("interface", - "LMyThreadSet;", + ASSERT_TRUE(HasMethod("LMyThreadSet;", "isEmpty", "()Z", - true, + /* expect_resolved */ true, "public", "Ljava/util/Set;")); } @@ -1058,23 +1046,25 @@ TEST_F(VerifierDepsTest, InvokeInterface_Resolved_DeclaredInSuperinterface2) { TEST_F(VerifierDepsTest, InvokeInterface_Unresolved1) { ASSERT_FALSE(VerifyMethod("InvokeInterface_Unresolved1")); ASSERT_TRUE(HasClass("Ljava/lang/Runnable;", true, "public interface")); - ASSERT_TRUE(HasMethod("interface", "Ljava/lang/Runnable;", "x", "()V", false)); + ASSERT_TRUE(HasMethod("Ljava/lang/Runnable;", + "x", + "()V", + /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeInterface_Unresolved2) { ASSERT_FALSE(VerifyMethod("InvokeInterface_Unresolved2")); - ASSERT_TRUE(HasMethod("interface", "LMyThreadSet;", "x", "()V", false)); + ASSERT_TRUE(HasMethod("LMyThreadSet;", "x", "()V", /* expect_resolved */ false)); } TEST_F(VerifierDepsTest, InvokeSuper_ThisAssignable) { ASSERT_TRUE(VerifyMethod("InvokeSuper_ThisAssignable")); ASSERT_TRUE(HasClass("Ljava/lang/Runnable;", true, "public interface")); ASSERT_TRUE(HasAssignable("Ljava/lang/Runnable;", "Ljava/lang/Thread;", true)); - ASSERT_TRUE(HasMethod("interface", - "Ljava/lang/Runnable;", + ASSERT_TRUE(HasMethod("Ljava/lang/Runnable;", "run", "()V", - true, + /* expect_resolved */ true, "public", "Ljava/lang/Runnable;")); } @@ -1083,8 +1073,10 @@ TEST_F(VerifierDepsTest, InvokeSuper_ThisNotAssignable) { ASSERT_FALSE(VerifyMethod("InvokeSuper_ThisNotAssignable")); ASSERT_TRUE(HasClass("Ljava/lang/Integer;", true, "public")); ASSERT_TRUE(HasAssignable("Ljava/lang/Integer;", "Ljava/lang/Thread;", false)); - ASSERT_TRUE(HasMethod( - "virtual", "Ljava/lang/Integer;", "intValue", "()I", true, "public", "Ljava/lang/Integer;")); + ASSERT_TRUE(HasMethod("Ljava/lang/Integer;", + "intValue", "()I", + /* expect_resolved */ true, + "public", "Ljava/lang/Integer;")); } TEST_F(VerifierDepsTest, ArgumentType_ResolvedReferenceArray) { @@ -1152,16 +1144,37 @@ TEST_F(VerifierDepsTest, UnverifiedClasses) { ASSERT_TRUE(HasUnverifiedClass("LMyClassWithNoSuperButFailures;")); } -// Returns the next resolution kind in the enum. -static MethodResolutionKind GetNextResolutionKind(MethodResolutionKind resolution_kind) { - if (resolution_kind == kDirectMethodResolution) { - return kVirtualMethodResolution; - } else if (resolution_kind == kVirtualMethodResolution) { - return kInterfaceMethodResolution; - } else { - DCHECK_EQ(resolution_kind, kInterfaceMethodResolution); - return kDirectMethodResolution; - } +TEST_F(VerifierDepsTest, UnverifiedOrder) { + ScopedObjectAccess soa(Thread::Current()); + jobject loader = LoadDex("VerifierDeps"); + std::vector<const DexFile*> dex_files = GetDexFiles(loader); + ASSERT_GT(dex_files.size(), 0u); + const DexFile* dex_file = dex_files[0]; + VerifierDeps deps1(dex_files); + Thread* const self = Thread::Current(); + ASSERT_TRUE(self->GetVerifierDeps() == nullptr); + self->SetVerifierDeps(&deps1); + deps1.MaybeRecordVerificationStatus(*dex_file, + dex::TypeIndex(0), + verifier::FailureKind::kHardFailure); + deps1.MaybeRecordVerificationStatus(*dex_file, + dex::TypeIndex(1), + verifier::FailureKind::kHardFailure); + VerifierDeps deps2(dex_files); + self->SetVerifierDeps(nullptr); + self->SetVerifierDeps(&deps2); + deps2.MaybeRecordVerificationStatus(*dex_file, + dex::TypeIndex(1), + verifier::FailureKind::kHardFailure); + deps2.MaybeRecordVerificationStatus(*dex_file, + dex::TypeIndex(0), + verifier::FailureKind::kHardFailure); + self->SetVerifierDeps(nullptr); + std::vector<uint8_t> buffer1; + deps1.Encode(dex_files, &buffer1); + std::vector<uint8_t> buffer2; + deps2.Encode(dex_files, &buffer2); + EXPECT_EQ(buffer1, buffer2); } TEST_F(VerifierDepsTest, VerifyDeps) { @@ -1340,131 +1353,82 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } // Mess up with methods. - for (MethodResolutionKind resolution_kind : - { kDirectMethodResolution, kVirtualMethodResolution, kInterfaceMethodResolution }) { - { - VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); - VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); - bool found = false; - std::set<VerifierDeps::MethodResolution>* methods = GetMethods(deps, resolution_kind); - for (const auto& entry : *methods) { - if (entry.IsResolved()) { - methods->insert(VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), - VerifierDeps::kUnresolvedMarker, - entry.GetDeclaringClassIndex())); - found = true; - break; - } - } - ASSERT_TRUE(found); - new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); - } - - { - VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); - VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); - bool found = false; - std::set<VerifierDeps::MethodResolution>* methods = GetMethods(deps, resolution_kind); - for (const auto& entry : *methods) { - if (!entry.IsResolved()) { - constexpr dex::StringIndex kStringIndexZero(0); // We know there is a class there. - methods->insert(VerifierDeps::MethodResolution(0 /* we know there is a method there */, - VerifierDeps::kUnresolvedMarker - 1, - kStringIndexZero)); - found = true; - break; - } - } - ASSERT_TRUE(found); - new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); - } - - { - VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); - VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); - bool found = false; - std::set<VerifierDeps::MethodResolution>* methods = GetMethods(deps, resolution_kind); - for (const auto& entry : *methods) { - if (entry.IsResolved()) { - methods->insert(VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), - entry.GetAccessFlags() - 1, - entry.GetDeclaringClassIndex())); - found = true; - break; - } + { + VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); + VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); + bool found = false; + std::set<VerifierDeps::MethodResolution>* methods = &deps->methods_; + for (const auto& entry : *methods) { + if (entry.IsResolved()) { + methods->insert(VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), + VerifierDeps::kUnresolvedMarker, + entry.GetDeclaringClassIndex())); + found = true; + break; } - ASSERT_TRUE(found); - new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } + ASSERT_TRUE(found); + new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); + } - { - VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); - VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); - bool found = false; - std::set<VerifierDeps::MethodResolution>* methods = GetMethods(deps, resolution_kind); - for (const auto& entry : *methods) { - constexpr dex::StringIndex kNewTypeIndex(0); - if (entry.IsResolved() && entry.GetDeclaringClassIndex() != kNewTypeIndex) { - methods->insert(VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), - entry.GetAccessFlags(), - kNewTypeIndex)); - found = true; - break; - } + { + VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); + VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); + bool found = false; + std::set<VerifierDeps::MethodResolution>* methods = &deps->methods_; + for (const auto& entry : *methods) { + if (!entry.IsResolved()) { + constexpr dex::StringIndex kStringIndexZero(0); // We know there is a class there. + methods->insert(VerifierDeps::MethodResolution(0 /* we know there is a method there */, + VerifierDeps::kUnresolvedMarker - 1, + kStringIndexZero)); + found = true; + break; } - ASSERT_TRUE(found); - new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } + ASSERT_TRUE(found); + new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); + } - // The two tests below make sure that fiddling with the method kind - // (static, virtual, interface) is detected by `ValidateDependencies`. - - // An interface method lookup can succeed with a virtual method lookup on the same class. - // That's OK, as we only want to make sure there is a method being defined with the right - // flags. Therefore, polluting the interface methods with virtual methods does not have - // to fail verification. - if (resolution_kind != kVirtualMethodResolution) { - VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); - VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); - bool found = false; - std::set<VerifierDeps::MethodResolution>* methods = GetMethods(deps, resolution_kind); - for (const auto& entry : *methods) { - if (entry.IsResolved()) { - GetMethods(deps, GetNextResolutionKind(resolution_kind))->insert( - VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), - entry.GetAccessFlags(), - entry.GetDeclaringClassIndex())); - found = true; - } + { + VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); + VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); + bool found = false; + std::set<VerifierDeps::MethodResolution>* methods = &deps->methods_; + for (const auto& entry : *methods) { + if (entry.IsResolved()) { + methods->insert(VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), + entry.GetAccessFlags() - 1, + entry.GetDeclaringClassIndex())); + found = true; + break; } - ASSERT_TRUE(found); - new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } + ASSERT_TRUE(found); + new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); + } - // See comment above that applies the same way. - if (resolution_kind != kInterfaceMethodResolution) { - VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); - VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); - bool found = false; - std::set<VerifierDeps::MethodResolution>* methods = GetMethods(deps, resolution_kind); - for (const auto& entry : *methods) { - if (entry.IsResolved()) { - GetMethods(deps, GetNextResolutionKind(GetNextResolutionKind(resolution_kind)))->insert( - VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), - entry.GetAccessFlags(), - entry.GetDeclaringClassIndex())); - found = true; - } + { + VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); + VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); + bool found = false; + std::set<VerifierDeps::MethodResolution>* methods = &deps->methods_; + for (const auto& entry : *methods) { + constexpr dex::StringIndex kNewTypeIndex(0); + if (entry.IsResolved() && entry.GetDeclaringClassIndex() != kNewTypeIndex) { + methods->insert(VerifierDeps::MethodResolution(entry.GetDexMethodIndex(), + entry.GetAccessFlags(), + kNewTypeIndex)); + found = true; + break; } - ASSERT_TRUE(found); - new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } + ASSERT_TRUE(found); + new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } } @@ -1510,7 +1474,6 @@ TEST_F(VerifierDepsTest, CompilerDriver) { ASSERT_FALSE(verifier_deps_ == nullptr); ASSERT_FALSE(verifier_deps_->Equals(decoded_deps)); } else { - ASSERT_TRUE(verifier_deps_ == nullptr); VerifyClassStatus(decoded_deps); } } |