diff options
author | 2017-11-23 16:05:19 +0000 | |
---|---|---|
committer | 2018-04-04 10:34:36 +0100 | |
commit | c9dd2207dfdab42586b1d6a5e7f11cf2fcea3a7a (patch) | |
tree | 879df31fd10658093b8931117ee617064ce82519 /compiler/linker | |
parent | 30a2d9c61da75359dee4ce90236d19fc6341b07a (diff) |
Compile link-time thunks in codegen.
Prepare for experimenting with Baker read barrier marking
introspection entrypoints for JIT.
Test: m test-art-host-gtest
Test: Compare compiled boot*.oat before and after (no diff).
Test: Pixel 2 XL boots.
Bug: 36141117
Change-Id: Idb413a31b158db4bf89a8707ea46dd167a06f110
Diffstat (limited to 'compiler/linker')
-rw-r--r-- | compiler/linker/arm/relative_patcher_arm_base.cc | 44 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_arm_base.h | 9 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_thumb2.cc | 305 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_thumb2.h | 78 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_thumb2_test.cc | 96 | ||||
-rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64.cc | 253 | ||||
-rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64.h | 54 | ||||
-rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64_test.cc | 87 | ||||
-rw-r--r-- | compiler/linker/linker_patch.h | 2 | ||||
-rw-r--r-- | compiler/linker/relative_patcher.cc | 13 | ||||
-rw-r--r-- | compiler/linker/relative_patcher.h | 27 | ||||
-rw-r--r-- | compiler/linker/relative_patcher_test.h | 72 |
12 files changed, 284 insertions, 756 deletions
diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc index 6e0286afac..7cb8ae55c5 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.cc +++ b/compiler/linker/arm/relative_patcher_arm_base.cc @@ -30,8 +30,9 @@ namespace linker { class ArmBaseRelativePatcher::ThunkData { public: - ThunkData(std::vector<uint8_t> code, uint32_t max_next_offset) - : code_(std::move(code)), + ThunkData(ArrayRef<const uint8_t> code, const std::string& debug_name, uint32_t max_next_offset) + : code_(code), + debug_name_(debug_name), offsets_(), max_next_offset_(max_next_offset), pending_offset_(0u) { @@ -45,7 +46,11 @@ class ArmBaseRelativePatcher::ThunkData { } ArrayRef<const uint8_t> GetCode() const { - return ArrayRef<const uint8_t>(code_); + return code_; + } + + const std::string& GetDebugName() const { + return debug_name_; } bool NeedsNextThunk() const { @@ -142,10 +147,11 @@ class ArmBaseRelativePatcher::ThunkData { } private: - std::vector<uint8_t> code_; // The code of the thunk. - std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written. - uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed. - uint32_t pending_offset_; // The index of the next offset to write. + const ArrayRef<const uint8_t> code_; // The code of the thunk. + const std::string debug_name_; // The debug name of the thunk. + std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written. + uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed. + uint32_t pending_offset_; // The index of the next offset to write. }; class ArmBaseRelativePatcher::PendingThunkComparator { @@ -239,14 +245,13 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn std::vector<debug::MethodDebugInfo> result; result.reserve(number_of_thunks); for (auto&& entry : thunks_) { - const ThunkKey& key = entry.first; const ThunkData& data = entry.second; size_t start = data.IndexOfFirstThunkAtOrAfter(executable_offset); if (start == data.NumberOfThunks()) { continue; } // Get the base name to use for the first occurrence of the thunk. - std::string base_name = GetThunkDebugName(key); + std::string base_name = data.GetDebugName(); for (size_t i = start, num = data.NumberOfThunks(); i != num; ++i) { debug::MethodDebugInfo info = {}; if (i == 0u) { @@ -267,9 +272,11 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn return result; } -ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, +ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, InstructionSet instruction_set) - : provider_(provider), + : thunk_provider_(thunk_provider), + target_provider_(target_provider), instruction_set_(instruction_set), thunks_(), unprocessed_method_call_patches_(), @@ -398,7 +405,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho unprocessed_method_call_patches_.emplace_back(patch_offset, patch.TargetMethod()); if (method_call_thunk_ == nullptr) { uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key); - auto it = thunks_.Put(key, ThunkData(CompileThunk(key), max_next_offset)); + auto it = thunks_.Put(key, ThunkDataForPatch(patch, max_next_offset)); method_call_thunk_ = &it->second; AddUnreservedThunk(method_call_thunk_); } else { @@ -409,7 +416,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho auto lb = thunks_.lower_bound(key); if (lb == thunks_.end() || thunks_.key_comp()(key, lb->first)) { uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key); - auto it = thunks_.PutBefore(lb, key, ThunkData(CompileThunk(key), max_next_offset)); + auto it = thunks_.PutBefore(lb, key, ThunkDataForPatch(patch, max_next_offset)); AddUnreservedThunk(&it->second); } else { old_data = &lb->second; @@ -477,7 +484,7 @@ void ArmBaseRelativePatcher::ResolveMethodCalls(uint32_t quick_code_offset, break; } } else { - auto result = provider_->FindMethodOffset(target_method); + auto result = target_provider_->FindMethodOffset(target_method); if (!result.first) { break; } @@ -518,5 +525,14 @@ inline uint32_t ArmBaseRelativePatcher::CalculateMaxNextOffset(uint32_t patch_of GetInstructionSetAlignment(instruction_set_)); } +inline ArmBaseRelativePatcher::ThunkData ArmBaseRelativePatcher::ThunkDataForPatch( + const LinkerPatch& patch, uint32_t max_next_offset) { + ArrayRef<const uint8_t> code; + std::string debug_name; + thunk_provider_->GetThunkCode(patch, &code, &debug_name); + DCHECK(!code.empty()); + return ThunkData(code, debug_name, max_next_offset); +} + } // namespace linker } // namespace art diff --git a/compiler/linker/arm/relative_patcher_arm_base.h b/compiler/linker/arm/relative_patcher_arm_base.h index ee09bf96b3..963d6690b0 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.h +++ b/compiler/linker/arm/relative_patcher_arm_base.h @@ -37,7 +37,8 @@ class ArmBaseRelativePatcher : public RelativePatcher { std::vector<debug::MethodDebugInfo> GenerateThunkDebugInfo(uint32_t executable_offset) OVERRIDE; protected: - ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, + ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, InstructionSet instruction_set); ~ArmBaseRelativePatcher(); @@ -94,8 +95,6 @@ class ArmBaseRelativePatcher : public RelativePatcher { uint32_t CalculateMethodCallDisplacement(uint32_t patch_offset, uint32_t target_offset); - virtual std::vector<uint8_t> CompileThunk(const ThunkKey& key) = 0; - virtual std::string GetThunkDebugName(const ThunkKey& key) = 0; virtual uint32_t MaxPositiveDisplacement(const ThunkKey& key) = 0; virtual uint32_t MaxNegativeDisplacement(const ThunkKey& key) = 0; @@ -108,8 +107,10 @@ class ArmBaseRelativePatcher : public RelativePatcher { void ResolveMethodCalls(uint32_t quick_code_offset, MethodReference method_ref); uint32_t CalculateMaxNextOffset(uint32_t patch_offset, const ThunkKey& key); + ThunkData ThunkDataForPatch(const LinkerPatch& patch, uint32_t max_next_offset); - RelativePatcherTargetProvider* const provider_; + RelativePatcherThunkProvider* const thunk_provider_; + RelativePatcherTargetProvider* const target_provider_; const InstructionSet instruction_set_; // The data for all thunks. diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc index 78755176e4..7400d11c31 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.cc +++ b/compiler/linker/arm/relative_patcher_thumb2.cc @@ -48,8 +48,9 @@ constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 24) - kPcDisplace constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 2u + kPcDisplacement; constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20) - kPcDisplacement; -Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherTargetProvider* provider) - : ArmBaseRelativePatcher(provider, InstructionSet::kThumb2) { +Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider) + : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kThumb2) { } void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code, @@ -110,62 +111,6 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co uint32_t insn = GetInsn32(code, literal_offset); DCHECK_EQ(insn, 0xf0408000); // BNE +0 (unpatched) ThunkKey key = GetBakerThunkKey(patch); - if (kIsDebugBuild) { - const uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - // Check that the next instruction matches the expected LDR. - switch (kind) { - case BakerReadBarrierKind::kField: { - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - if (width == BakerReadBarrierWidth::kWide) { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn32(code, literal_offset + 4u); - // LDR (immediate), encoding T3, with correct base_reg. - CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16)); - } else { - DCHECK_GE(code->size() - literal_offset, 6u); - uint32_t next_insn = GetInsn16(code, literal_offset + 4u); - // LDR (immediate), encoding T1, with correct base_reg. - CheckValidReg(next_insn & 0x7u); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3)); - } - break; - } - case BakerReadBarrierKind::kArray: { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn32(code, literal_offset + 4u); - // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]). - CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16)); - CheckValidReg(next_insn & 0xf); // Check index register - break; - } - case BakerReadBarrierKind::kGcRoot: { - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - if (width == BakerReadBarrierWidth::kWide) { - DCHECK_GE(literal_offset, 4u); - uint32_t prev_insn = GetInsn32(code, literal_offset - 4u); - // LDR (immediate), encoding T3, with correct root_reg. - const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12)); - } else { - DCHECK_GE(literal_offset, 2u); - uint32_t prev_insn = GetInsn16(code, literal_offset - 2u); - // LDR (immediate), encoding T1, with correct root_reg. - const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg); - } - break; - } - default: - LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(key.GetType()); - UNREACHABLE(); - } - } uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); DCHECK_ALIGNED(target_offset, 4u); uint32_t disp = target_offset - (patch_offset + kPcDisplacement); @@ -178,250 +123,6 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co SetInsn32(code, literal_offset, insn); } -#define __ assembler.GetVIXLAssembler()-> - -static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler, - vixl::aarch32::Register base_reg, - vixl::aarch32::MemOperand& lock_word, - vixl::aarch32::Label* slow_path, - int32_t raw_ldr_offset) { - using namespace vixl::aarch32; // NOLINT(build/namespaces) - // Load the lock word containing the rb_state. - __ Ldr(ip, lock_word); - // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted)); - __ B(ne, slow_path, /* is_far_target */ false); - __ Add(lr, lr, raw_ldr_offset); - // Introduce a dependency on the lock_word including rb_state, - // to prevent load-load reordering, and without using - // a memory barrier (which would be more expensive). - __ Add(base_reg, base_reg, Operand(ip, LSR, 32)); - __ Bx(lr); // And return back to the function. - // Note: The fake dependency is unnecessary for the slow path. -} - -// Load the read barrier introspection entrypoint in register `entrypoint` -static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler, - vixl::aarch32::Register entrypoint) { - using vixl::aarch32::MemOperand; - using vixl::aarch32::ip; - // Thread Register. - const vixl::aarch32::Register tr = vixl::aarch32::r9; - - // The register where the read barrier introspection entrypoint is loaded - // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4). - DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister); - // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); -} - -void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, - uint32_t encoded_data) { - using namespace vixl::aarch32; // NOLINT(build/namespaces) - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - switch (kind) { - case BakerReadBarrierKind::kField: { - // Check if the holder is gray and, if not, add fake dependency to the base register - // and return to the LDR instruction to load the reference. Otherwise, use introspection - // to load the reference and call the entrypoint (in kBakerCcEntrypointRegister) - // that performs further checks on the reference and marks it if needed. - Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data)); - CheckValidReg(holder_reg.GetCode()); - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip); - // If base_reg differs from holder_reg, the offset was too large and we must have - // emitted an explicit null check before the load. Otherwise, we need to null-check - // the holder as we do not necessarily do that check before going to the thunk. - vixl::aarch32::Label throw_npe; - if (holder_reg.Is(base_reg)) { - __ CompareAndBranchIfZero(holder_reg, &throw_npe, /* is_far_target */ false); - } - vixl::aarch32::Label slow_path; - MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); - const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide) - ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET - : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET; - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset); - __ Bind(&slow_path); - const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + - raw_ldr_offset; - Register ep_reg(kBakerCcEntrypointRegister); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); - if (width == BakerReadBarrierWidth::kWide) { - MemOperand ldr_half_address(lr, ldr_offset + 2); - __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12". - __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12. - __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference. - } else { - MemOperand ldr_address(lr, ldr_offset); - __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1. - __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint - ep_reg, // for narrow LDR. - Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET)); - __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4. - __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference. - } - // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. - __ Bx(ep_reg); // Jump to the entrypoint. - if (holder_reg.Is(base_reg)) { - // Add null check slow path. The stack map is at the address pointed to by LR. - __ Bind(&throw_npe); - int32_t offset = GetThreadOffset<kArmPointerSize>(kQuickThrowNullPointer).Int32Value(); - __ Ldr(ip, MemOperand(/* Thread* */ vixl::aarch32::r9, offset)); - __ Bx(ip); - } - break; - } - case BakerReadBarrierKind::kArray: { - Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip); - vixl::aarch32::Label slow_path; - int32_t data_offset = - mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); - MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); - DCHECK_LT(lock_word.GetOffsetImmediate(), 0); - const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET; - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset); - __ Bind(&slow_path); - const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + - raw_ldr_offset; - MemOperand ldr_address(lr, ldr_offset + 2); - __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm", - // i.e. Rm+32 because the scale in imm2 is 2. - Register ep_reg(kBakerCcEntrypointRegister); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); - __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create - // a switch case target based on the index register. - __ Mov(ip, base_reg); // Move the base register to ip0. - __ Bx(ep_reg); // Jump to the entrypoint's array switch case. - break; - } - case BakerReadBarrierKind::kGcRoot: { - // Check if the reference needs to be marked and if so (i.e. not null, not marked yet - // and it does not have a forwarding address), call the correct introspection entrypoint; - // otherwise return the reference (or the extracted forwarding address). - // There is no gray bit check for GC roots. - Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(root_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip); - vixl::aarch32::Label return_label, not_marked, forwarding_address; - __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false); - MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value()); - __ Ldr(ip, lock_word); - __ Tst(ip, LockWord::kMarkBitStateMaskShifted); - __ B(eq, ¬_marked); - __ Bind(&return_label); - __ Bx(lr); - __ Bind(¬_marked); - static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3, - "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in " - " the highest bits and the 'forwarding address' state to have all bits set"); - __ Cmp(ip, Operand(0xc0000000)); - __ B(hs, &forwarding_address); - Register ep_reg(kBakerCcEntrypointRegister); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); - // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister - // to art_quick_read_barrier_mark_introspection_gc_roots. - int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide) - ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET - : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET; - __ Add(ep_reg, ep_reg, Operand(entrypoint_offset)); - __ Mov(ip, root_reg); - __ Bx(ep_reg); - __ Bind(&forwarding_address); - __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift); - __ Bx(lr); - break; - } - default: - LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); - UNREACHABLE(); - } -} - -std::vector<uint8_t> Thumb2RelativePatcher::CompileThunk(const ThunkKey& key) { - ArenaPool pool; - ArenaAllocator allocator(&pool); - arm::ArmVIXLAssembler assembler(&allocator); - - switch (key.GetType()) { - case ThunkType::kMethodCall: - // The thunk just uses the entry point in the ArtMethod. This works even for calls - // to the generic JNI and interpreter trampolines. - assembler.LoadFromOffset( - arm::kLoadWord, - vixl::aarch32::pc, - vixl::aarch32::r0, - ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); - __ Bkpt(0); - break; - case ThunkType::kBakerReadBarrier: - CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1()); - break; - } - - assembler.FinalizeCode(); - std::vector<uint8_t> thunk_code(assembler.CodeSize()); - MemoryRegion code(thunk_code.data(), thunk_code.size()); - assembler.FinalizeInstructions(code); - return thunk_code; -} - -std::string Thumb2RelativePatcher::GetThunkDebugName(const ThunkKey& key) { - switch (key.GetType()) { - case ThunkType::kMethodCall: - return "MethodCallThunk"; - - case ThunkType::kBakerReadBarrier: { - uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - std::ostringstream oss; - oss << "BakerReadBarrierThunk"; - switch (kind) { - case BakerReadBarrierKind::kField: - oss << "Field"; - if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { - oss << "Wide"; - } - oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) - << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); - break; - case BakerReadBarrierKind::kArray: - oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); - break; - case BakerReadBarrierKind::kGcRoot: - oss << "GcRoot"; - if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { - oss << "Wide"; - } - oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - break; - } - return oss.str(); - } - } -} - -#undef __ - uint32_t Thumb2RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { switch (key.GetType()) { case ThunkType::kMethodCall: diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h index 68386c00f4..68610d69e1 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.h +++ b/compiler/linker/arm/relative_patcher_thumb2.h @@ -19,8 +19,6 @@ #include "arch/arm/registers_arm.h" #include "base/array_ref.h" -#include "base/bit_field.h" -#include "base/bit_utils.h" #include "linker/arm/relative_patcher_arm_base.h" namespace art { @@ -33,42 +31,8 @@ namespace linker { class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { public: - static constexpr uint32_t kBakerCcEntrypointRegister = 4u; - - static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, - uint32_t holder_reg, - bool narrow) { - CheckValidReg(base_reg); - CheckValidReg(holder_reg); - DCHECK(!narrow || base_reg < 8u) << base_reg; - BakerReadBarrierWidth width = - narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(holder_reg) | - BakerReadBarrierWidthField::Encode(width); - } - - static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { - CheckValidReg(base_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) | - BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide); - } - - static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) { - CheckValidReg(root_reg); - DCHECK(!narrow || root_reg < 8u) << root_reg; - BakerReadBarrierWidth width = - narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | - BakerReadBarrierFirstRegField::Encode(root_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) | - BakerReadBarrierWidthField::Encode(width); - } - - explicit Thumb2RelativePatcher(RelativePatcherTargetProvider* provider); + explicit Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider); void PatchCall(std::vector<uint8_t>* code, uint32_t literal_offset, @@ -83,48 +47,10 @@ class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { uint32_t patch_offset) OVERRIDE; protected: - std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE; - std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE; uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE; uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE; private: - static constexpr uint32_t kInvalidEncodedReg = /* pc is invalid */ 15u; - - enum class BakerReadBarrierKind : uint8_t { - kField, // Field get or array get with constant offset (i.e. constant index). - kArray, // Array get with index in register. - kGcRoot, // GC root load. - kLast = kGcRoot - }; - - enum class BakerReadBarrierWidth : uint8_t { - kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled). - kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled). - kLast = kNarrow - }; - - static constexpr size_t kBitsForBakerReadBarrierKind = - MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); - static constexpr size_t kBitsForRegister = 4u; - using BakerReadBarrierKindField = - BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; - using BakerReadBarrierFirstRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>; - using BakerReadBarrierSecondRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>; - static constexpr size_t kBitsForBakerReadBarrierWidth = - MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast)); - using BakerReadBarrierWidthField = BitField<BakerReadBarrierWidth, - kBitsForBakerReadBarrierKind + 2 * kBitsForRegister, - kBitsForBakerReadBarrierWidth>; - - static void CheckValidReg(uint32_t reg) { - DCHECK(reg < 12u && reg != kBakerCcEntrypointRegister) << reg; - } - - void CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data); - void SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value); static uint32_t GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset); diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc index 2c22a352c2..e7b11bd16b 100644 --- a/compiler/linker/arm/relative_patcher_thumb2_test.cc +++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc @@ -16,12 +16,15 @@ #include "linker/arm/relative_patcher_thumb2.h" +#include "arch/arm/instruction_set_features_arm.h" #include "base/casts.h" #include "linker/relative_patcher_test.h" #include "lock_word.h" #include "mirror/array-inl.h" #include "mirror/object.h" #include "oat_quick_method_header.h" +#include "optimizing/code_generator_arm_vixl.h" +#include "optimizing/optimizing_unit_test.h" namespace art { namespace linker { @@ -189,9 +192,42 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { return result.second - 1 /* thumb mode */; } + std::vector<uint8_t> CompileThunk(const LinkerPatch& patch, + /*out*/ std::string* debug_name = nullptr) { + OptimizingUnitTestHelper helper; + HGraph* graph = helper.CreateGraph(); + std::string error_msg; + ArmFeaturesUniquePtr features = + ArmInstructionSetFeatures::FromVariant("default", &error_msg); + CompilerOptions options; + arm::CodeGeneratorARMVIXL codegen(graph, *features, options); + ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter()); + codegen.EmitThunkCode(patch, &code, debug_name); + return std::vector<uint8_t>(code.begin(), code.end()); + } + + void AddCompiledMethod( + MethodReference method_ref, + const ArrayRef<const uint8_t>& code, + const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) { + RelativePatcherTest::AddCompiledMethod(method_ref, code, patches); + + // Make sure the ThunkProvider has all the necessary thunks. + for (const LinkerPatch& patch : patches) { + if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == LinkerPatch::Type::kCallRelative) { + std::string debug_name; + std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name); + thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name); + } + } + } + std::vector<uint8_t> CompileMethodCallThunk() { - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey(); - return static_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u, + /* target_dex_file*/ nullptr, + /* target_method_idx */ 0u); + return CompileThunk(patch); } uint32_t MethodCallThunkSize() { @@ -228,27 +264,38 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { void TestStringReference(uint32_t string_offset); void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset); + static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, + uint32_t holder_reg, + bool narrow) { + return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow); + } + + static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg); + } + + static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) { + return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow); + } + std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg, bool narrow) { const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow)); + return CompileThunk(patch); } uint32_t GetOutputInsn32(uint32_t offset) { @@ -594,7 +641,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( base_reg, holder_reg, /* narrow */ false); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), @@ -696,7 +743,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t r const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( base_reg, holder_reg, /* narrow */ true); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), @@ -809,7 +856,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) { constexpr uint32_t kLiteralOffset1 = 6u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -877,7 +924,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) { constexpr uint32_t kLiteralOffset1 = 4u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -907,7 +954,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast constexpr uint32_t kLiteralOffset1 = 6u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -993,7 +1040,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerArray) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)), + kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1074,8 +1121,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)), + kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1134,8 +1180,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)), + kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1182,8 +1227,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) { patches.reserve(num_patches); const uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12); - uint32_t encoded_data = - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false); + uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false); for (size_t i = 0; i != num_patches; ++i) { PushBackInsn(&code, ldr); PushBackInsn(&code, kBneWPlus0); @@ -1264,10 +1308,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) { ldr1, kBneWPlus0, // First GC root LDR with read barrier. ldr2, kBneWPlus0, // Second GC root LDR with read barrier. }); - uint32_t encoded_data1 = - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false); - uint32_t encoded_data2 = - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false); + uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false); + uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false); const LinkerPatch last_method_patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1), LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2), diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index b268204b4a..135e39d100 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -82,9 +82,10 @@ inline uint32_t MaxExtraSpace(size_t num_adrp, size_t code_size) { } // anonymous namespace -Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, +Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, const Arm64InstructionSetFeatures* features) - : ArmBaseRelativePatcher(provider, InstructionSet::kArm64), + : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kArm64), fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), reserved_adrp_thunks_(0u), processed_adrp_thunks_(0u) { @@ -313,44 +314,6 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod uint32_t insn = GetInsn(code, literal_offset); DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched) ThunkKey key = GetBakerThunkKey(patch); - if (kIsDebugBuild) { - const uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - // Check that the next instruction matches the expected LDR. - switch (kind) { - case BakerReadBarrierKind::kField: { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn(code, literal_offset + 4u); - // LDR (immediate) with correct base_reg. - CheckValidReg(next_insn & 0x1fu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5)); - break; - } - case BakerReadBarrierKind::kArray: { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn(code, literal_offset + 4u); - // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL), - // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2]. - CheckValidReg(next_insn & 0x1fu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5)); - CheckValidReg((next_insn >> 16) & 0x1f); // Check index register - break; - } - case BakerReadBarrierKind::kGcRoot: { - DCHECK_GE(literal_offset, 4u); - uint32_t prev_insn = GetInsn(code, literal_offset - 4u); - // LDR (immediate) with correct root_reg. - const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg); - break; - } - default: - LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); - UNREACHABLE(); - } - } uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); DCHECK_ALIGNED(target_offset, 4u); uint32_t disp = target_offset - patch_offset; @@ -359,216 +322,6 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod SetInsn(code, literal_offset, insn); } -#define __ assembler.GetVIXLAssembler()-> - -static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler, - vixl::aarch64::Register base_reg, - vixl::aarch64::MemOperand& lock_word, - vixl::aarch64::Label* slow_path) { - using namespace vixl::aarch64; // NOLINT(build/namespaces) - // Load the lock word containing the rb_state. - __ Ldr(ip0.W(), lock_word); - // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path); - static_assert( - BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET, - "Field and array LDR offsets must be the same to reuse the same code."); - // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning). - static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), - "Field LDR must be 1 instruction (4B) before the return address label; " - " 2 instructions (8B) for heap poisoning."); - __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); - // Introduce a dependency on the lock_word including rb_state, - // to prevent load-load reordering, and without using - // a memory barrier (which would be more expensive). - __ Add(base_reg, base_reg, Operand(ip0, LSR, 32)); - __ Br(lr); // And return back to the function. - // Note: The fake dependency is unnecessary for the slow path. -} - -// Load the read barrier introspection entrypoint in register `entrypoint`. -static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler, - vixl::aarch64::Register entrypoint) { - using vixl::aarch64::MemOperand; - using vixl::aarch64::ip0; - // Thread Register. - const vixl::aarch64::Register tr = vixl::aarch64::x19; - - // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); -} - -void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, - uint32_t encoded_data) { - using namespace vixl::aarch64; // NOLINT(build/namespaces) - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - switch (kind) { - case BakerReadBarrierKind::kField: { - // Check if the holder is gray and, if not, add fake dependency to the base register - // and return to the LDR instruction to load the reference. Otherwise, use introspection - // to load the reference and call the entrypoint (in IP1) that performs further checks - // on the reference and marks it if needed. - auto base_reg = - Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - auto holder_reg = - Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data)); - CheckValidReg(holder_reg.GetCode()); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip0, ip1); - // If base_reg differs from holder_reg, the offset was too large and we must have - // emitted an explicit null check before the load. Otherwise, we need to null-check - // the holder as we do not necessarily do that check before going to the thunk. - vixl::aarch64::Label throw_npe; - if (holder_reg.Is(base_reg)) { - __ Cbz(holder_reg.W(), &throw_npe); - } - vixl::aarch64::Label slow_path; - MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); - __ Bind(&slow_path); - MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); - __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset. - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); - __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset. - __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference. - // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. - __ Br(ip1); // Jump to the entrypoint. - if (holder_reg.Is(base_reg)) { - // Add null check slow path. The stack map is at the address pointed to by LR. - __ Bind(&throw_npe); - int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value(); - __ Ldr(ip0, MemOperand(/* Thread* */ vixl::aarch64::x19, offset)); - __ Br(ip0); - } - break; - } - case BakerReadBarrierKind::kArray: { - auto base_reg = - Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip0, ip1); - vixl::aarch64::Label slow_path; - int32_t data_offset = - mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); - MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); - DCHECK_LT(lock_word.GetOffset(), 0); - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); - __ Bind(&slow_path); - MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); - __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset. - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); - __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set). - __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create - // a switch case target based on the index register. - __ Mov(ip0, base_reg); // Move the base register to ip0. - __ Br(ip1); // Jump to the entrypoint's array switch case. - break; - } - case BakerReadBarrierKind::kGcRoot: { - // Check if the reference needs to be marked and if so (i.e. not null, not marked yet - // and it does not have a forwarding address), call the correct introspection entrypoint; - // otherwise return the reference (or the extracted forwarding address). - // There is no gray bit check for GC roots. - auto root_reg = - Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(root_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip0, ip1); - vixl::aarch64::Label return_label, not_marked, forwarding_address; - __ Cbz(root_reg, &return_label); - MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value()); - __ Ldr(ip0.W(), lock_word); - __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, ¬_marked); - __ Bind(&return_label); - __ Br(lr); - __ Bind(¬_marked); - __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1)); - __ B(&forwarding_address, mi); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); - // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to - // art_quick_read_barrier_mark_introspection_gc_roots. - __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET)); - __ Mov(ip0.W(), root_reg); - __ Br(ip1); - __ Bind(&forwarding_address); - __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift); - __ Br(lr); - break; - } - default: - LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); - UNREACHABLE(); - } -} - -std::vector<uint8_t> Arm64RelativePatcher::CompileThunk(const ThunkKey& key) { - ArenaPool pool; - ArenaAllocator allocator(&pool); - arm64::Arm64Assembler assembler(&allocator); - - switch (key.GetType()) { - case ThunkType::kMethodCall: { - // The thunk just uses the entry point in the ArtMethod. This works even for calls - // to the generic JNI and interpreter trampolines. - Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArm64PointerSize).Int32Value()); - assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); - break; - } - case ThunkType::kBakerReadBarrier: { - CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1()); - break; - } - } - - // Ensure we emit the literal pool. - assembler.FinalizeCode(); - std::vector<uint8_t> thunk_code(assembler.CodeSize()); - MemoryRegion code(thunk_code.data(), thunk_code.size()); - assembler.FinalizeInstructions(code); - return thunk_code; -} - -std::string Arm64RelativePatcher::GetThunkDebugName(const ThunkKey& key) { - switch (key.GetType()) { - case ThunkType::kMethodCall: - return "MethodCallThunk"; - - case ThunkType::kBakerReadBarrier: { - uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - std::ostringstream oss; - oss << "BakerReadBarrierThunk"; - switch (kind) { - case BakerReadBarrierKind::kField: - oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) - << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); - break; - case BakerReadBarrierKind::kArray: - oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - break; - case BakerReadBarrierKind::kGcRoot: - oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - break; - } - return oss.str(); - } - } -} - -#undef __ - uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { switch (key.GetType()) { case ThunkType::kMethodCall: diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h index 8ba59976e7..9dc289da44 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.h +++ b/compiler/linker/arm64/relative_patcher_arm64.h @@ -18,8 +18,6 @@ #define ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ #include "base/array_ref.h" -#include "base/bit_field.h" -#include "base/bit_utils.h" #include "linker/arm/relative_patcher_arm_base.h" namespace art { @@ -32,29 +30,8 @@ namespace linker { class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { public: - static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { - CheckValidReg(base_reg); - CheckValidReg(holder_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(holder_reg); - } - - static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { - CheckValidReg(base_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg); - } - - static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { - CheckValidReg(root_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | - BakerReadBarrierFirstRegField::Encode(root_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg); - } - - Arm64RelativePatcher(RelativePatcherTargetProvider* provider, + Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, const Arm64InstructionSetFeatures* features); uint32_t ReserveSpace(uint32_t offset, @@ -75,37 +52,10 @@ class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { uint32_t patch_offset) OVERRIDE; protected: - std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE; - std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE; uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE; uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE; private: - static constexpr uint32_t kInvalidEncodedReg = /* sp/zr is invalid */ 31u; - - enum class BakerReadBarrierKind : uint8_t { - kField, // Field get or array get with constant offset (i.e. constant index). - kArray, // Array get with index in register. - kGcRoot, // GC root load. - kLast = kGcRoot - }; - - static constexpr size_t kBitsForBakerReadBarrierKind = - MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); - static constexpr size_t kBitsForRegister = 5u; - using BakerReadBarrierKindField = - BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; - using BakerReadBarrierFirstRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>; - using BakerReadBarrierSecondRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>; - - static void CheckValidReg(uint32_t reg) { - DCHECK(reg < 30u && reg != 16u && reg != 17u) << reg; - } - - void CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data); - static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp); static bool NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, uint32_t literal_offset, diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc index 05459a2a82..393733dd0c 100644 --- a/compiler/linker/arm64/relative_patcher_arm64_test.cc +++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc @@ -16,12 +16,15 @@ #include "linker/arm64/relative_patcher_arm64.h" +#include "arch/arm64/instruction_set_features_arm64.h" #include "base/casts.h" #include "linker/relative_patcher_test.h" #include "lock_word.h" #include "mirror/array-inl.h" #include "mirror/object.h" #include "oat_quick_method_header.h" +#include "optimizing/code_generator_arm64.h" +#include "optimizing/optimizing_unit_test.h" namespace art { namespace linker { @@ -168,9 +171,42 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { return result.second; } + std::vector<uint8_t> CompileThunk(const LinkerPatch& patch, + /*out*/ std::string* debug_name = nullptr) { + OptimizingUnitTestHelper helper; + HGraph* graph = helper.CreateGraph(); + std::string error_msg; + Arm64FeaturesUniquePtr features = + Arm64InstructionSetFeatures::FromVariant("default", &error_msg); + CompilerOptions options; + arm64::CodeGeneratorARM64 codegen(graph, *features, options); + ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter()); + codegen.EmitThunkCode(patch, &code, debug_name); + return std::vector<uint8_t>(code.begin(), code.end()); + } + + void AddCompiledMethod( + MethodReference method_ref, + const ArrayRef<const uint8_t>& code, + const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) { + RelativePatcherTest::AddCompiledMethod(method_ref, code, patches); + + // Make sure the ThunkProvider has all the necessary thunks. + for (const LinkerPatch& patch : patches) { + if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == LinkerPatch::Type::kCallRelative) { + std::string debug_name; + std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name); + thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name); + } + } + } + std::vector<uint8_t> CompileMethodCallThunk() { - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey(); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u, + /* target_dex_file*/ nullptr, + /* target_method_idx */ 0u); + return CompileThunk(patch); } uint32_t MethodCallThunkSize() { @@ -475,25 +511,34 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset); } + static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { + return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierFieldData(base_reg, holder_reg); + } + + static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierArrayData(base_reg); + } + + static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { + return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierGcRootData(root_reg); + } + std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) { const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg)); + return CompileThunk(patch); } uint32_t GetOutputInsn(uint32_t offset) { @@ -919,8 +964,7 @@ void Arm64RelativePatcherTest::TestBakerField(uint32_t offset, uint32_t ref_reg) const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(base_reg, holder_reg); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), }; @@ -1005,8 +1049,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddle) { constexpr uint32_t kLiteralOffset1 = 4; const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1066,8 +1109,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkBeforeFiller) { constexpr uint32_t kLiteralOffset1 = 0; const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1096,8 +1138,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddleUnreachableFr constexpr uint32_t kLiteralOffset1 = 4; const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1170,7 +1211,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerArray) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)), + kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1247,7 +1288,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerGcRoot) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)), + kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1343,8 +1384,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerAndMethodCallInteraction) { kNopInsn, kNopInsn, // Padding before second GC root read barrier. ldr2, kCbnzIP1Plus0Insn, // Second GC root LDR with read barrier. }); - uint32_t encoded_data1 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1); - uint32_t encoded_data2 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2); + uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1); + uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2); const LinkerPatch last_method_patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1), LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2), diff --git a/compiler/linker/linker_patch.h b/compiler/linker/linker_patch.h index 710d8a690a..7b35fd9b0c 100644 --- a/compiler/linker/linker_patch.h +++ b/compiler/linker/linker_patch.h @@ -141,7 +141,7 @@ class LinkerPatch { static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset, uint32_t custom_value1 = 0u, uint32_t custom_value2 = 0u) { - LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, nullptr); + LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, /* target_dex_file */ nullptr); patch.baker_custom_value1_ = custom_value1; patch.baker_custom_value2_ = custom_value2; return patch; diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc index 13877f8f12..b82d15230d 100644 --- a/compiler/linker/relative_patcher.cc +++ b/compiler/linker/relative_patcher.cc @@ -43,7 +43,8 @@ namespace linker { std::unique_ptr<RelativePatcher> RelativePatcher::Create( InstructionSet instruction_set, const InstructionSetFeatures* features, - RelativePatcherTargetProvider* provider) { + RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider) { class RelativePatcherNone FINAL : public RelativePatcher { public: RelativePatcherNone() { } @@ -92,7 +93,8 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create( }; UNUSED(features); - UNUSED(provider); + UNUSED(thunk_provider); + UNUSED(target_provider); switch (instruction_set) { #ifdef ART_ENABLE_CODEGEN_x86 case InstructionSet::kX86: @@ -106,12 +108,15 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create( case InstructionSet::kArm: // Fall through: we generate Thumb2 code for "arm". case InstructionSet::kThumb2: - return std::unique_ptr<RelativePatcher>(new Thumb2RelativePatcher(provider)); + return std::unique_ptr<RelativePatcher>( + new Thumb2RelativePatcher(thunk_provider, target_provider)); #endif #ifdef ART_ENABLE_CODEGEN_arm64 case InstructionSet::kArm64: return std::unique_ptr<RelativePatcher>( - new Arm64RelativePatcher(provider, features->AsArm64InstructionSetFeatures())); + new Arm64RelativePatcher(thunk_provider, + target_provider, + features->AsArm64InstructionSetFeatures())); #endif #ifdef ART_ENABLE_CODEGEN_mips case InstructionSet::kMips: diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h index b58e3dffbd..06c7e70d23 100644 --- a/compiler/linker/relative_patcher.h +++ b/compiler/linker/relative_patcher.h @@ -39,6 +39,27 @@ class LinkerPatch; class OutputStream; /** + * @class RelativePatcherThunkProvider + * @brief Interface for providing method offsets for relative call targets. + */ +class RelativePatcherThunkProvider { + public: + /** + * Get the code and debug name of a thunk needed by the given linker patch. + * + * @param patch The patch for which we need to retrieve the thunk code. + * @param code A variable to receive the code of the thunk. This code must not be empty. + * @param debug_name A variable to receive the debug name of the thunk. + */ + virtual void GetThunkCode(const LinkerPatch& patch, + /*out*/ ArrayRef<const uint8_t>* code, + /*out*/ std::string* debug_name) = 0; + + protected: + virtual ~RelativePatcherThunkProvider() { } +}; + +/** * @class RelativePatcherTargetProvider * @brief Interface for providing method offsets for relative call targets. */ @@ -70,8 +91,10 @@ class RelativePatcherTargetProvider { class RelativePatcher { public: static std::unique_ptr<RelativePatcher> Create( - InstructionSet instruction_set, const InstructionSetFeatures* features, - RelativePatcherTargetProvider* provider); + InstructionSet instruction_set, + const InstructionSetFeatures* features, + RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider); virtual ~RelativePatcher() { } diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h index d21f2795b9..af8dc4dbc9 100644 --- a/compiler/linker/relative_patcher_test.h +++ b/compiler/linker/relative_patcher_test.h @@ -58,7 +58,10 @@ class RelativePatcherTest : public testing::Test { instruction_set_(instruction_set), features_(InstructionSetFeatures::FromVariant(instruction_set, variant, &error_msg_)), method_offset_map_(), - patcher_(RelativePatcher::Create(instruction_set, features_.get(), &method_offset_map_)), + patcher_(RelativePatcher::Create(instruction_set, + features_.get(), + &thunk_provider_, + &method_offset_map_)), bss_begin_(0u), compiled_method_refs_(), compiled_methods_(), @@ -248,6 +251,72 @@ class RelativePatcherTest : public testing::Test { LOG(ERROR) << " " << diff_indicator_str; } + class ThunkProvider : public RelativePatcherThunkProvider { + public: + ThunkProvider() {} + + void SetThunkCode(const LinkerPatch& patch, + ArrayRef<const uint8_t> code, + const std::string& debug_name) { + thunk_map_.emplace(ThunkKey(patch), ThunkValue(code, debug_name)); + } + + void GetThunkCode(const LinkerPatch& patch, + /*out*/ ArrayRef<const uint8_t>* code, + /*out*/ std::string* debug_name) OVERRIDE { + auto it = thunk_map_.find(ThunkKey(patch)); + CHECK(it != thunk_map_.end()); + const ThunkValue& value = it->second; + CHECK(code != nullptr); + *code = value.GetCode(); + CHECK(debug_name != nullptr); + *debug_name = value.GetDebugName(); + } + + private: + class ThunkKey { + public: + explicit ThunkKey(const LinkerPatch& patch) + : type_(patch.GetType()), + custom_value1_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch + ? patch.GetBakerCustomValue1() : 0u), + custom_value2_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch + ? patch.GetBakerCustomValue2() : 0u) { + CHECK(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == LinkerPatch::Type::kCallRelative); + } + + bool operator<(const ThunkKey& other) const { + if (custom_value1_ != other.custom_value1_) { + return custom_value1_ < other.custom_value1_; + } + if (custom_value2_ != other.custom_value2_) { + return custom_value2_ < other.custom_value2_; + } + return type_ < other.type_; + } + + private: + const LinkerPatch::Type type_; + const uint32_t custom_value1_; + const uint32_t custom_value2_; + }; + + class ThunkValue { + public: + ThunkValue(ArrayRef<const uint8_t> code, const std::string& debug_name) + : code_(code.begin(), code.end()), debug_name_(debug_name) {} + ArrayRef<const uint8_t> GetCode() const { return ArrayRef<const uint8_t>(code_); } + const std::string& GetDebugName() const { return debug_name_; } + + private: + const std::vector<uint8_t> code_; + const std::string debug_name_; + }; + + std::map<ThunkKey, ThunkValue> thunk_map_; + }; + // Map method reference to assinged offset. // Wrap the map in a class implementing RelativePatcherTargetProvider. class MethodOffsetMap FINAL : public RelativePatcherTargetProvider { @@ -272,6 +341,7 @@ class RelativePatcherTest : public testing::Test { std::string error_msg_; InstructionSet instruction_set_; std::unique_ptr<const InstructionSetFeatures> features_; + ThunkProvider thunk_provider_; MethodOffsetMap method_offset_map_; std::unique_ptr<RelativePatcher> patcher_; uint32_t bss_begin_; |