diff options
author | 2018-04-04 17:23:28 +0000 | |
---|---|---|
committer | 2018-04-04 17:23:28 +0000 | |
commit | 5806a9ec99b5494b511e84c74f494f0b3a8ebec5 (patch) | |
tree | bb50d00ff0890c2e10f351f462b47b56b01e78ea /compiler/linker | |
parent | c9dd2207dfdab42586b1d6a5e7f11cf2fcea3a7a (diff) |
Revert "Compile link-time thunks in codegen."
Reason for revert: This caused clang linker crash
in several branches.
Bug: 77581732
This reverts commit c9dd2207dfdab42586b1d6a5e7f11cf2fcea3a7a.
Change-Id: I1923809083cf41c4f19e3e60df03ae80517aaedb
Diffstat (limited to 'compiler/linker')
-rw-r--r-- | compiler/linker/arm/relative_patcher_arm_base.cc | 44 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_arm_base.h | 9 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_thumb2.cc | 305 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_thumb2.h | 78 | ||||
-rw-r--r-- | compiler/linker/arm/relative_patcher_thumb2_test.cc | 96 | ||||
-rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64.cc | 253 | ||||
-rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64.h | 54 | ||||
-rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64_test.cc | 87 | ||||
-rw-r--r-- | compiler/linker/linker_patch.h | 2 | ||||
-rw-r--r-- | compiler/linker/relative_patcher.cc | 13 | ||||
-rw-r--r-- | compiler/linker/relative_patcher.h | 27 | ||||
-rw-r--r-- | compiler/linker/relative_patcher_test.h | 72 |
12 files changed, 756 insertions, 284 deletions
diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc index 7cb8ae55c5..6e0286afac 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.cc +++ b/compiler/linker/arm/relative_patcher_arm_base.cc @@ -30,9 +30,8 @@ namespace linker { class ArmBaseRelativePatcher::ThunkData { public: - ThunkData(ArrayRef<const uint8_t> code, const std::string& debug_name, uint32_t max_next_offset) - : code_(code), - debug_name_(debug_name), + ThunkData(std::vector<uint8_t> code, uint32_t max_next_offset) + : code_(std::move(code)), offsets_(), max_next_offset_(max_next_offset), pending_offset_(0u) { @@ -46,11 +45,7 @@ class ArmBaseRelativePatcher::ThunkData { } ArrayRef<const uint8_t> GetCode() const { - return code_; - } - - const std::string& GetDebugName() const { - return debug_name_; + return ArrayRef<const uint8_t>(code_); } bool NeedsNextThunk() const { @@ -147,11 +142,10 @@ class ArmBaseRelativePatcher::ThunkData { } private: - const ArrayRef<const uint8_t> code_; // The code of the thunk. - const std::string debug_name_; // The debug name of the thunk. - std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written. - uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed. - uint32_t pending_offset_; // The index of the next offset to write. + std::vector<uint8_t> code_; // The code of the thunk. + std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written. + uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed. + uint32_t pending_offset_; // The index of the next offset to write. }; class ArmBaseRelativePatcher::PendingThunkComparator { @@ -245,13 +239,14 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn std::vector<debug::MethodDebugInfo> result; result.reserve(number_of_thunks); for (auto&& entry : thunks_) { + const ThunkKey& key = entry.first; const ThunkData& data = entry.second; size_t start = data.IndexOfFirstThunkAtOrAfter(executable_offset); if (start == data.NumberOfThunks()) { continue; } // Get the base name to use for the first occurrence of the thunk. - std::string base_name = data.GetDebugName(); + std::string base_name = GetThunkDebugName(key); for (size_t i = start, num = data.NumberOfThunks(); i != num; ++i) { debug::MethodDebugInfo info = {}; if (i == 0u) { @@ -272,11 +267,9 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn return result; } -ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider, +ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, InstructionSet instruction_set) - : thunk_provider_(thunk_provider), - target_provider_(target_provider), + : provider_(provider), instruction_set_(instruction_set), thunks_(), unprocessed_method_call_patches_(), @@ -405,7 +398,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho unprocessed_method_call_patches_.emplace_back(patch_offset, patch.TargetMethod()); if (method_call_thunk_ == nullptr) { uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key); - auto it = thunks_.Put(key, ThunkDataForPatch(patch, max_next_offset)); + auto it = thunks_.Put(key, ThunkData(CompileThunk(key), max_next_offset)); method_call_thunk_ = &it->second; AddUnreservedThunk(method_call_thunk_); } else { @@ -416,7 +409,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho auto lb = thunks_.lower_bound(key); if (lb == thunks_.end() || thunks_.key_comp()(key, lb->first)) { uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key); - auto it = thunks_.PutBefore(lb, key, ThunkDataForPatch(patch, max_next_offset)); + auto it = thunks_.PutBefore(lb, key, ThunkData(CompileThunk(key), max_next_offset)); AddUnreservedThunk(&it->second); } else { old_data = &lb->second; @@ -484,7 +477,7 @@ void ArmBaseRelativePatcher::ResolveMethodCalls(uint32_t quick_code_offset, break; } } else { - auto result = target_provider_->FindMethodOffset(target_method); + auto result = provider_->FindMethodOffset(target_method); if (!result.first) { break; } @@ -525,14 +518,5 @@ inline uint32_t ArmBaseRelativePatcher::CalculateMaxNextOffset(uint32_t patch_of GetInstructionSetAlignment(instruction_set_)); } -inline ArmBaseRelativePatcher::ThunkData ArmBaseRelativePatcher::ThunkDataForPatch( - const LinkerPatch& patch, uint32_t max_next_offset) { - ArrayRef<const uint8_t> code; - std::string debug_name; - thunk_provider_->GetThunkCode(patch, &code, &debug_name); - DCHECK(!code.empty()); - return ThunkData(code, debug_name, max_next_offset); -} - } // namespace linker } // namespace art diff --git a/compiler/linker/arm/relative_patcher_arm_base.h b/compiler/linker/arm/relative_patcher_arm_base.h index 963d6690b0..ee09bf96b3 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.h +++ b/compiler/linker/arm/relative_patcher_arm_base.h @@ -37,8 +37,7 @@ class ArmBaseRelativePatcher : public RelativePatcher { std::vector<debug::MethodDebugInfo> GenerateThunkDebugInfo(uint32_t executable_offset) OVERRIDE; protected: - ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider, + ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, InstructionSet instruction_set); ~ArmBaseRelativePatcher(); @@ -95,6 +94,8 @@ class ArmBaseRelativePatcher : public RelativePatcher { uint32_t CalculateMethodCallDisplacement(uint32_t patch_offset, uint32_t target_offset); + virtual std::vector<uint8_t> CompileThunk(const ThunkKey& key) = 0; + virtual std::string GetThunkDebugName(const ThunkKey& key) = 0; virtual uint32_t MaxPositiveDisplacement(const ThunkKey& key) = 0; virtual uint32_t MaxNegativeDisplacement(const ThunkKey& key) = 0; @@ -107,10 +108,8 @@ class ArmBaseRelativePatcher : public RelativePatcher { void ResolveMethodCalls(uint32_t quick_code_offset, MethodReference method_ref); uint32_t CalculateMaxNextOffset(uint32_t patch_offset, const ThunkKey& key); - ThunkData ThunkDataForPatch(const LinkerPatch& patch, uint32_t max_next_offset); - RelativePatcherThunkProvider* const thunk_provider_; - RelativePatcherTargetProvider* const target_provider_; + RelativePatcherTargetProvider* const provider_; const InstructionSet instruction_set_; // The data for all thunks. diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc index 7400d11c31..78755176e4 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.cc +++ b/compiler/linker/arm/relative_patcher_thumb2.cc @@ -48,9 +48,8 @@ constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 24) - kPcDisplace constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 2u + kPcDisplacement; constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20) - kPcDisplacement; -Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider) - : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kThumb2) { +Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherTargetProvider* provider) + : ArmBaseRelativePatcher(provider, InstructionSet::kThumb2) { } void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code, @@ -111,6 +110,62 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co uint32_t insn = GetInsn32(code, literal_offset); DCHECK_EQ(insn, 0xf0408000); // BNE +0 (unpatched) ThunkKey key = GetBakerThunkKey(patch); + if (kIsDebugBuild) { + const uint32_t encoded_data = key.GetCustomValue1(); + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + // Check that the next instruction matches the expected LDR. + switch (kind) { + case BakerReadBarrierKind::kField: { + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + if (width == BakerReadBarrierWidth::kWide) { + DCHECK_GE(code->size() - literal_offset, 8u); + uint32_t next_insn = GetInsn32(code, literal_offset + 4u); + // LDR (immediate), encoding T3, with correct base_reg. + CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16)); + } else { + DCHECK_GE(code->size() - literal_offset, 6u); + uint32_t next_insn = GetInsn16(code, literal_offset + 4u); + // LDR (immediate), encoding T1, with correct base_reg. + CheckValidReg(next_insn & 0x7u); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3)); + } + break; + } + case BakerReadBarrierKind::kArray: { + DCHECK_GE(code->size() - literal_offset, 8u); + uint32_t next_insn = GetInsn32(code, literal_offset + 4u); + // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]). + CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16)); + CheckValidReg(next_insn & 0xf); // Check index register + break; + } + case BakerReadBarrierKind::kGcRoot: { + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + if (width == BakerReadBarrierWidth::kWide) { + DCHECK_GE(literal_offset, 4u); + uint32_t prev_insn = GetInsn32(code, literal_offset - 4u); + // LDR (immediate), encoding T3, with correct root_reg. + const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12)); + } else { + DCHECK_GE(literal_offset, 2u); + uint32_t prev_insn = GetInsn16(code, literal_offset - 2u); + // LDR (immediate), encoding T1, with correct root_reg. + const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg); + } + break; + } + default: + LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(key.GetType()); + UNREACHABLE(); + } + } uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); DCHECK_ALIGNED(target_offset, 4u); uint32_t disp = target_offset - (patch_offset + kPcDisplacement); @@ -123,6 +178,250 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co SetInsn32(code, literal_offset, insn); } +#define __ assembler.GetVIXLAssembler()-> + +static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler, + vixl::aarch32::Register base_reg, + vixl::aarch32::MemOperand& lock_word, + vixl::aarch32::Label* slow_path, + int32_t raw_ldr_offset) { + using namespace vixl::aarch32; // NOLINT(build/namespaces) + // Load the lock word containing the rb_state. + __ Ldr(ip, lock_word); + // Given the numeric representation, it's enough to check the low bit of the rb_state. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted)); + __ B(ne, slow_path, /* is_far_target */ false); + __ Add(lr, lr, raw_ldr_offset); + // Introduce a dependency on the lock_word including rb_state, + // to prevent load-load reordering, and without using + // a memory barrier (which would be more expensive). + __ Add(base_reg, base_reg, Operand(ip, LSR, 32)); + __ Bx(lr); // And return back to the function. + // Note: The fake dependency is unnecessary for the slow path. +} + +// Load the read barrier introspection entrypoint in register `entrypoint` +static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler, + vixl::aarch32::Register entrypoint) { + using vixl::aarch32::MemOperand; + using vixl::aarch32::ip; + // Thread Register. + const vixl::aarch32::Register tr = vixl::aarch32::r9; + + // The register where the read barrier introspection entrypoint is loaded + // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4). + DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister); + // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip.GetCode(), 12u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + +void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, + uint32_t encoded_data) { + using namespace vixl::aarch32; // NOLINT(build/namespaces) + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + switch (kind) { + case BakerReadBarrierKind::kField: { + // Check if the holder is gray and, if not, add fake dependency to the base register + // and return to the LDR instruction to load the reference. Otherwise, use introspection + // to load the reference and call the entrypoint (in kBakerCcEntrypointRegister) + // that performs further checks on the reference and marks it if needed. + Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data)); + CheckValidReg(holder_reg.GetCode()); + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip); + // If base_reg differs from holder_reg, the offset was too large and we must have + // emitted an explicit null check before the load. Otherwise, we need to null-check + // the holder as we do not necessarily do that check before going to the thunk. + vixl::aarch32::Label throw_npe; + if (holder_reg.Is(base_reg)) { + __ CompareAndBranchIfZero(holder_reg, &throw_npe, /* is_far_target */ false); + } + vixl::aarch32::Label slow_path; + MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); + const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide) + ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET + : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET; + EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset); + __ Bind(&slow_path); + const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + + raw_ldr_offset; + Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + if (width == BakerReadBarrierWidth::kWide) { + MemOperand ldr_half_address(lr, ldr_offset + 2); + __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12". + __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12. + __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference. + } else { + MemOperand ldr_address(lr, ldr_offset); + __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1. + __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint + ep_reg, // for narrow LDR. + Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET)); + __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4. + __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference. + } + // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. + __ Bx(ep_reg); // Jump to the entrypoint. + if (holder_reg.Is(base_reg)) { + // Add null check slow path. The stack map is at the address pointed to by LR. + __ Bind(&throw_npe); + int32_t offset = GetThreadOffset<kArmPointerSize>(kQuickThrowNullPointer).Int32Value(); + __ Ldr(ip, MemOperand(/* Thread* */ vixl::aarch32::r9, offset)); + __ Bx(ip); + } + break; + } + case BakerReadBarrierKind::kArray: { + Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip); + vixl::aarch32::Label slow_path; + int32_t data_offset = + mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); + MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); + DCHECK_LT(lock_word.GetOffsetImmediate(), 0); + const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET; + EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset); + __ Bind(&slow_path); + const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + + raw_ldr_offset; + MemOperand ldr_address(lr, ldr_offset + 2); + __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm", + // i.e. Rm+32 because the scale in imm2 is 2. + Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create + // a switch case target based on the index register. + __ Mov(ip, base_reg); // Move the base register to ip0. + __ Bx(ep_reg); // Jump to the entrypoint's array switch case. + break; + } + case BakerReadBarrierKind::kGcRoot: { + // Check if the reference needs to be marked and if so (i.e. not null, not marked yet + // and it does not have a forwarding address), call the correct introspection entrypoint; + // otherwise return the reference (or the extracted forwarding address). + // There is no gray bit check for GC roots. + Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(root_reg.GetCode()); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip); + vixl::aarch32::Label return_label, not_marked, forwarding_address; + __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false); + MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value()); + __ Ldr(ip, lock_word); + __ Tst(ip, LockWord::kMarkBitStateMaskShifted); + __ B(eq, ¬_marked); + __ Bind(&return_label); + __ Bx(lr); + __ Bind(¬_marked); + static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3, + "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in " + " the highest bits and the 'forwarding address' state to have all bits set"); + __ Cmp(ip, Operand(0xc0000000)); + __ B(hs, &forwarding_address); + Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister + // to art_quick_read_barrier_mark_introspection_gc_roots. + int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide) + ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET + : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET; + __ Add(ep_reg, ep_reg, Operand(entrypoint_offset)); + __ Mov(ip, root_reg); + __ Bx(ep_reg); + __ Bind(&forwarding_address); + __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift); + __ Bx(lr); + break; + } + default: + LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); + UNREACHABLE(); + } +} + +std::vector<uint8_t> Thumb2RelativePatcher::CompileThunk(const ThunkKey& key) { + ArenaPool pool; + ArenaAllocator allocator(&pool); + arm::ArmVIXLAssembler assembler(&allocator); + + switch (key.GetType()) { + case ThunkType::kMethodCall: + // The thunk just uses the entry point in the ArtMethod. This works even for calls + // to the generic JNI and interpreter trampolines. + assembler.LoadFromOffset( + arm::kLoadWord, + vixl::aarch32::pc, + vixl::aarch32::r0, + ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); + __ Bkpt(0); + break; + case ThunkType::kBakerReadBarrier: + CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1()); + break; + } + + assembler.FinalizeCode(); + std::vector<uint8_t> thunk_code(assembler.CodeSize()); + MemoryRegion code(thunk_code.data(), thunk_code.size()); + assembler.FinalizeInstructions(code); + return thunk_code; +} + +std::string Thumb2RelativePatcher::GetThunkDebugName(const ThunkKey& key) { + switch (key.GetType()) { + case ThunkType::kMethodCall: + return "MethodCallThunk"; + + case ThunkType::kBakerReadBarrier: { + uint32_t encoded_data = key.GetCustomValue1(); + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + std::ostringstream oss; + oss << "BakerReadBarrierThunk"; + switch (kind) { + case BakerReadBarrierKind::kField: + oss << "Field"; + if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { + oss << "Wide"; + } + oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) + << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); + break; + case BakerReadBarrierKind::kArray: + oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); + break; + case BakerReadBarrierKind::kGcRoot: + oss << "GcRoot"; + if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { + oss << "Wide"; + } + oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + break; + } + return oss.str(); + } + } +} + +#undef __ + uint32_t Thumb2RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { switch (key.GetType()) { case ThunkType::kMethodCall: diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h index 68610d69e1..68386c00f4 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.h +++ b/compiler/linker/arm/relative_patcher_thumb2.h @@ -19,6 +19,8 @@ #include "arch/arm/registers_arm.h" #include "base/array_ref.h" +#include "base/bit_field.h" +#include "base/bit_utils.h" #include "linker/arm/relative_patcher_arm_base.h" namespace art { @@ -31,8 +33,42 @@ namespace linker { class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { public: - explicit Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider); + static constexpr uint32_t kBakerCcEntrypointRegister = 4u; + + static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, + uint32_t holder_reg, + bool narrow) { + CheckValidReg(base_reg); + CheckValidReg(holder_reg); + DCHECK(!narrow || base_reg < 8u) << base_reg; + BakerReadBarrierWidth width = + narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(holder_reg) | + BakerReadBarrierWidthField::Encode(width); + } + + static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + CheckValidReg(base_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) | + BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide); + } + + static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) { + CheckValidReg(root_reg); + DCHECK(!narrow || root_reg < 8u) << root_reg; + BakerReadBarrierWidth width = + narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | + BakerReadBarrierFirstRegField::Encode(root_reg) | + BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) | + BakerReadBarrierWidthField::Encode(width); + } + + explicit Thumb2RelativePatcher(RelativePatcherTargetProvider* provider); void PatchCall(std::vector<uint8_t>* code, uint32_t literal_offset, @@ -47,10 +83,48 @@ class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { uint32_t patch_offset) OVERRIDE; protected: + std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE; + std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE; uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE; uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE; private: + static constexpr uint32_t kInvalidEncodedReg = /* pc is invalid */ 15u; + + enum class BakerReadBarrierKind : uint8_t { + kField, // Field get or array get with constant offset (i.e. constant index). + kArray, // Array get with index in register. + kGcRoot, // GC root load. + kLast = kGcRoot + }; + + enum class BakerReadBarrierWidth : uint8_t { + kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled). + kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled). + kLast = kNarrow + }; + + static constexpr size_t kBitsForBakerReadBarrierKind = + MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); + static constexpr size_t kBitsForRegister = 4u; + using BakerReadBarrierKindField = + BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; + using BakerReadBarrierFirstRegField = + BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>; + using BakerReadBarrierSecondRegField = + BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>; + static constexpr size_t kBitsForBakerReadBarrierWidth = + MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast)); + using BakerReadBarrierWidthField = BitField<BakerReadBarrierWidth, + kBitsForBakerReadBarrierKind + 2 * kBitsForRegister, + kBitsForBakerReadBarrierWidth>; + + static void CheckValidReg(uint32_t reg) { + DCHECK(reg < 12u && reg != kBakerCcEntrypointRegister) << reg; + } + + void CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data); + void SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value); static uint32_t GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset); diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc index e7b11bd16b..2c22a352c2 100644 --- a/compiler/linker/arm/relative_patcher_thumb2_test.cc +++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc @@ -16,15 +16,12 @@ #include "linker/arm/relative_patcher_thumb2.h" -#include "arch/arm/instruction_set_features_arm.h" #include "base/casts.h" #include "linker/relative_patcher_test.h" #include "lock_word.h" #include "mirror/array-inl.h" #include "mirror/object.h" #include "oat_quick_method_header.h" -#include "optimizing/code_generator_arm_vixl.h" -#include "optimizing/optimizing_unit_test.h" namespace art { namespace linker { @@ -192,42 +189,9 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { return result.second - 1 /* thumb mode */; } - std::vector<uint8_t> CompileThunk(const LinkerPatch& patch, - /*out*/ std::string* debug_name = nullptr) { - OptimizingUnitTestHelper helper; - HGraph* graph = helper.CreateGraph(); - std::string error_msg; - ArmFeaturesUniquePtr features = - ArmInstructionSetFeatures::FromVariant("default", &error_msg); - CompilerOptions options; - arm::CodeGeneratorARMVIXL codegen(graph, *features, options); - ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter()); - codegen.EmitThunkCode(patch, &code, debug_name); - return std::vector<uint8_t>(code.begin(), code.end()); - } - - void AddCompiledMethod( - MethodReference method_ref, - const ArrayRef<const uint8_t>& code, - const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) { - RelativePatcherTest::AddCompiledMethod(method_ref, code, patches); - - // Make sure the ThunkProvider has all the necessary thunks. - for (const LinkerPatch& patch : patches) { - if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || - patch.GetType() == LinkerPatch::Type::kCallRelative) { - std::string debug_name; - std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name); - thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name); - } - } - } - std::vector<uint8_t> CompileMethodCallThunk() { - LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u, - /* target_dex_file*/ nullptr, - /* target_method_idx */ 0u); - return CompileThunk(patch); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey(); + return static_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); } uint32_t MethodCallThunkSize() { @@ -264,38 +228,27 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { void TestStringReference(uint32_t string_offset); void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset); - static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, - uint32_t holder_reg, - bool narrow) { - return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow); - } - - static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { - return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg); - } - - static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) { - return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow); - } - std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg, bool narrow) { const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow)); - return CompileThunk(patch); + 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow)); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); + return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); } std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg)); - return CompileThunk(patch); + 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); + return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); } std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow)); - return CompileThunk(patch); + 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow)); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); + return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); } uint32_t GetOutputInsn32(uint32_t offset) { @@ -641,7 +594,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( base_reg, holder_reg, /* narrow */ false); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), @@ -743,7 +696,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t r const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( base_reg, holder_reg, /* narrow */ true); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), @@ -856,7 +809,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) { constexpr uint32_t kLiteralOffset1 = 6u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -924,7 +877,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) { constexpr uint32_t kLiteralOffset1 = 4u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -954,7 +907,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast constexpr uint32_t kLiteralOffset1 = 6u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -1040,7 +993,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerArray) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)), + kLiteralOffset, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1121,7 +1074,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)), + kLiteralOffset, + Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1180,7 +1134,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)), + kLiteralOffset, + Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1227,7 +1182,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) { patches.reserve(num_patches); const uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12); - uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false); + uint32_t encoded_data = + Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false); for (size_t i = 0; i != num_patches; ++i) { PushBackInsn(&code, ldr); PushBackInsn(&code, kBneWPlus0); @@ -1308,8 +1264,10 @@ TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) { ldr1, kBneWPlus0, // First GC root LDR with read barrier. ldr2, kBneWPlus0, // Second GC root LDR with read barrier. }); - uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false); - uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false); + uint32_t encoded_data1 = + Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false); + uint32_t encoded_data2 = + Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false); const LinkerPatch last_method_patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1), LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2), diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index 135e39d100..b268204b4a 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -82,10 +82,9 @@ inline uint32_t MaxExtraSpace(size_t num_adrp, size_t code_size) { } // anonymous namespace -Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider, +Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, const Arm64InstructionSetFeatures* features) - : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kArm64), + : ArmBaseRelativePatcher(provider, InstructionSet::kArm64), fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), reserved_adrp_thunks_(0u), processed_adrp_thunks_(0u) { @@ -314,6 +313,44 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod uint32_t insn = GetInsn(code, literal_offset); DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched) ThunkKey key = GetBakerThunkKey(patch); + if (kIsDebugBuild) { + const uint32_t encoded_data = key.GetCustomValue1(); + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + // Check that the next instruction matches the expected LDR. + switch (kind) { + case BakerReadBarrierKind::kField: { + DCHECK_GE(code->size() - literal_offset, 8u); + uint32_t next_insn = GetInsn(code, literal_offset + 4u); + // LDR (immediate) with correct base_reg. + CheckValidReg(next_insn & 0x1fu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5)); + break; + } + case BakerReadBarrierKind::kArray: { + DCHECK_GE(code->size() - literal_offset, 8u); + uint32_t next_insn = GetInsn(code, literal_offset + 4u); + // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL), + // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2]. + CheckValidReg(next_insn & 0x1fu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5)); + CheckValidReg((next_insn >> 16) & 0x1f); // Check index register + break; + } + case BakerReadBarrierKind::kGcRoot: { + DCHECK_GE(literal_offset, 4u); + uint32_t prev_insn = GetInsn(code, literal_offset - 4u); + // LDR (immediate) with correct root_reg. + const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg); + break; + } + default: + LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); + UNREACHABLE(); + } + } uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); DCHECK_ALIGNED(target_offset, 4u); uint32_t disp = target_offset - patch_offset; @@ -322,6 +359,216 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod SetInsn(code, literal_offset, insn); } +#define __ assembler.GetVIXLAssembler()-> + +static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler, + vixl::aarch64::Register base_reg, + vixl::aarch64::MemOperand& lock_word, + vixl::aarch64::Label* slow_path) { + using namespace vixl::aarch64; // NOLINT(build/namespaces) + // Load the lock word containing the rb_state. + __ Ldr(ip0.W(), lock_word); + // Given the numeric representation, it's enough to check the low bit of the rb_state. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path); + static_assert( + BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET, + "Field and array LDR offsets must be the same to reuse the same code."); + // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning). + static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), + "Field LDR must be 1 instruction (4B) before the return address label; " + " 2 instructions (8B) for heap poisoning."); + __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); + // Introduce a dependency on the lock_word including rb_state, + // to prevent load-load reordering, and without using + // a memory barrier (which would be more expensive). + __ Add(base_reg, base_reg, Operand(ip0, LSR, 32)); + __ Br(lr); // And return back to the function. + // Note: The fake dependency is unnecessary for the slow path. +} + +// Load the read barrier introspection entrypoint in register `entrypoint`. +static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler, + vixl::aarch64::Register entrypoint) { + using vixl::aarch64::MemOperand; + using vixl::aarch64::ip0; + // Thread Register. + const vixl::aarch64::Register tr = vixl::aarch64::x19; + + // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip0.GetCode(), 16u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + +void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, + uint32_t encoded_data) { + using namespace vixl::aarch64; // NOLINT(build/namespaces) + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + switch (kind) { + case BakerReadBarrierKind::kField: { + // Check if the holder is gray and, if not, add fake dependency to the base register + // and return to the LDR instruction to load the reference. Otherwise, use introspection + // to load the reference and call the entrypoint (in IP1) that performs further checks + // on the reference and marks it if needed. + auto base_reg = + Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + auto holder_reg = + Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data)); + CheckValidReg(holder_reg.GetCode()); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip0, ip1); + // If base_reg differs from holder_reg, the offset was too large and we must have + // emitted an explicit null check before the load. Otherwise, we need to null-check + // the holder as we do not necessarily do that check before going to the thunk. + vixl::aarch64::Label throw_npe; + if (holder_reg.Is(base_reg)) { + __ Cbz(holder_reg.W(), &throw_npe); + } + vixl::aarch64::Label slow_path; + MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); + EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); + __ Bind(&slow_path); + MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); + __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); + __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset. + __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference. + // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. + __ Br(ip1); // Jump to the entrypoint. + if (holder_reg.Is(base_reg)) { + // Add null check slow path. The stack map is at the address pointed to by LR. + __ Bind(&throw_npe); + int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value(); + __ Ldr(ip0, MemOperand(/* Thread* */ vixl::aarch64::x19, offset)); + __ Br(ip0); + } + break; + } + case BakerReadBarrierKind::kArray: { + auto base_reg = + Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip0, ip1); + vixl::aarch64::Label slow_path; + int32_t data_offset = + mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); + MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); + DCHECK_LT(lock_word.GetOffset(), 0); + EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); + __ Bind(&slow_path); + MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); + __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); + __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set). + __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create + // a switch case target based on the index register. + __ Mov(ip0, base_reg); // Move the base register to ip0. + __ Br(ip1); // Jump to the entrypoint's array switch case. + break; + } + case BakerReadBarrierKind::kGcRoot: { + // Check if the reference needs to be marked and if so (i.e. not null, not marked yet + // and it does not have a forwarding address), call the correct introspection entrypoint; + // otherwise return the reference (or the extracted forwarding address). + // There is no gray bit check for GC roots. + auto root_reg = + Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(root_reg.GetCode()); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip0, ip1); + vixl::aarch64::Label return_label, not_marked, forwarding_address; + __ Cbz(root_reg, &return_label); + MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value()); + __ Ldr(ip0.W(), lock_word); + __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, ¬_marked); + __ Bind(&return_label); + __ Br(lr); + __ Bind(¬_marked); + __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1)); + __ B(&forwarding_address, mi); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); + // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to + // art_quick_read_barrier_mark_introspection_gc_roots. + __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET)); + __ Mov(ip0.W(), root_reg); + __ Br(ip1); + __ Bind(&forwarding_address); + __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift); + __ Br(lr); + break; + } + default: + LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); + UNREACHABLE(); + } +} + +std::vector<uint8_t> Arm64RelativePatcher::CompileThunk(const ThunkKey& key) { + ArenaPool pool; + ArenaAllocator allocator(&pool); + arm64::Arm64Assembler assembler(&allocator); + + switch (key.GetType()) { + case ThunkType::kMethodCall: { + // The thunk just uses the entry point in the ArtMethod. This works even for calls + // to the generic JNI and interpreter trampolines. + Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kArm64PointerSize).Int32Value()); + assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); + break; + } + case ThunkType::kBakerReadBarrier: { + CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1()); + break; + } + } + + // Ensure we emit the literal pool. + assembler.FinalizeCode(); + std::vector<uint8_t> thunk_code(assembler.CodeSize()); + MemoryRegion code(thunk_code.data(), thunk_code.size()); + assembler.FinalizeInstructions(code); + return thunk_code; +} + +std::string Arm64RelativePatcher::GetThunkDebugName(const ThunkKey& key) { + switch (key.GetType()) { + case ThunkType::kMethodCall: + return "MethodCallThunk"; + + case ThunkType::kBakerReadBarrier: { + uint32_t encoded_data = key.GetCustomValue1(); + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + std::ostringstream oss; + oss << "BakerReadBarrierThunk"; + switch (kind) { + case BakerReadBarrierKind::kField: + oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) + << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); + break; + case BakerReadBarrierKind::kArray: + oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + break; + case BakerReadBarrierKind::kGcRoot: + oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); + break; + } + return oss.str(); + } + } +} + +#undef __ + uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { switch (key.GetType()) { case ThunkType::kMethodCall: diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h index 9dc289da44..8ba59976e7 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.h +++ b/compiler/linker/arm64/relative_patcher_arm64.h @@ -18,6 +18,8 @@ #define ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ #include "base/array_ref.h" +#include "base/bit_field.h" +#include "base/bit_utils.h" #include "linker/arm/relative_patcher_arm_base.h" namespace art { @@ -30,8 +32,29 @@ namespace linker { class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { public: - Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider, + static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { + CheckValidReg(base_reg); + CheckValidReg(holder_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(holder_reg); + } + + static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + CheckValidReg(base_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg); + } + + static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { + CheckValidReg(root_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | + BakerReadBarrierFirstRegField::Encode(root_reg) | + BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg); + } + + Arm64RelativePatcher(RelativePatcherTargetProvider* provider, const Arm64InstructionSetFeatures* features); uint32_t ReserveSpace(uint32_t offset, @@ -52,10 +75,37 @@ class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { uint32_t patch_offset) OVERRIDE; protected: + std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE; + std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE; uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE; uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE; private: + static constexpr uint32_t kInvalidEncodedReg = /* sp/zr is invalid */ 31u; + + enum class BakerReadBarrierKind : uint8_t { + kField, // Field get or array get with constant offset (i.e. constant index). + kArray, // Array get with index in register. + kGcRoot, // GC root load. + kLast = kGcRoot + }; + + static constexpr size_t kBitsForBakerReadBarrierKind = + MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); + static constexpr size_t kBitsForRegister = 5u; + using BakerReadBarrierKindField = + BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; + using BakerReadBarrierFirstRegField = + BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>; + using BakerReadBarrierSecondRegField = + BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>; + + static void CheckValidReg(uint32_t reg) { + DCHECK(reg < 30u && reg != 16u && reg != 17u) << reg; + } + + void CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data); + static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp); static bool NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, uint32_t literal_offset, diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc index 393733dd0c..05459a2a82 100644 --- a/compiler/linker/arm64/relative_patcher_arm64_test.cc +++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc @@ -16,15 +16,12 @@ #include "linker/arm64/relative_patcher_arm64.h" -#include "arch/arm64/instruction_set_features_arm64.h" #include "base/casts.h" #include "linker/relative_patcher_test.h" #include "lock_word.h" #include "mirror/array-inl.h" #include "mirror/object.h" #include "oat_quick_method_header.h" -#include "optimizing/code_generator_arm64.h" -#include "optimizing/optimizing_unit_test.h" namespace art { namespace linker { @@ -171,42 +168,9 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { return result.second; } - std::vector<uint8_t> CompileThunk(const LinkerPatch& patch, - /*out*/ std::string* debug_name = nullptr) { - OptimizingUnitTestHelper helper; - HGraph* graph = helper.CreateGraph(); - std::string error_msg; - Arm64FeaturesUniquePtr features = - Arm64InstructionSetFeatures::FromVariant("default", &error_msg); - CompilerOptions options; - arm64::CodeGeneratorARM64 codegen(graph, *features, options); - ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter()); - codegen.EmitThunkCode(patch, &code, debug_name); - return std::vector<uint8_t>(code.begin(), code.end()); - } - - void AddCompiledMethod( - MethodReference method_ref, - const ArrayRef<const uint8_t>& code, - const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) { - RelativePatcherTest::AddCompiledMethod(method_ref, code, patches); - - // Make sure the ThunkProvider has all the necessary thunks. - for (const LinkerPatch& patch : patches) { - if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || - patch.GetType() == LinkerPatch::Type::kCallRelative) { - std::string debug_name; - std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name); - thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name); - } - } - } - std::vector<uint8_t> CompileMethodCallThunk() { - LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u, - /* target_dex_file*/ nullptr, - /* target_method_idx */ 0u); - return CompileThunk(patch); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey(); + return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); } uint32_t MethodCallThunkSize() { @@ -511,34 +475,25 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset); } - static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { - return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierFieldData(base_reg, holder_reg); - } - - static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { - return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierArrayData(base_reg); - } - - static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { - return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierGcRootData(root_reg); - } - std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) { const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg)); - return CompileThunk(patch); + 0u, Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg)); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); + return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); } std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg)); - return CompileThunk(patch); + 0u, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); + return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); } std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg)); - return CompileThunk(patch); + 0u, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)); + ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); + return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); } uint32_t GetOutputInsn(uint32_t offset) { @@ -964,7 +919,8 @@ void Arm64RelativePatcherTest::TestBakerField(uint32_t offset, uint32_t ref_reg) const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData(base_reg, holder_reg); + uint32_t encoded_data = + Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), }; @@ -1049,7 +1005,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddle) { constexpr uint32_t kLiteralOffset1 = 4; const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = + Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1109,7 +1066,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkBeforeFiller) { constexpr uint32_t kLiteralOffset1 = 0; const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = + Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1138,7 +1096,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddleUnreachableFr constexpr uint32_t kLiteralOffset1 = 4; const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = + Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1211,7 +1170,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerArray) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)), + kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1288,7 +1247,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerGcRoot) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg)), + kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1384,8 +1343,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerAndMethodCallInteraction) { kNopInsn, kNopInsn, // Padding before second GC root read barrier. ldr2, kCbnzIP1Plus0Insn, // Second GC root LDR with read barrier. }); - uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1); - uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2); + uint32_t encoded_data1 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1); + uint32_t encoded_data2 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2); const LinkerPatch last_method_patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1), LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2), diff --git a/compiler/linker/linker_patch.h b/compiler/linker/linker_patch.h index 7b35fd9b0c..710d8a690a 100644 --- a/compiler/linker/linker_patch.h +++ b/compiler/linker/linker_patch.h @@ -141,7 +141,7 @@ class LinkerPatch { static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset, uint32_t custom_value1 = 0u, uint32_t custom_value2 = 0u) { - LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, /* target_dex_file */ nullptr); + LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, nullptr); patch.baker_custom_value1_ = custom_value1; patch.baker_custom_value2_ = custom_value2; return patch; diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc index b82d15230d..13877f8f12 100644 --- a/compiler/linker/relative_patcher.cc +++ b/compiler/linker/relative_patcher.cc @@ -43,8 +43,7 @@ namespace linker { std::unique_ptr<RelativePatcher> RelativePatcher::Create( InstructionSet instruction_set, const InstructionSetFeatures* features, - RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider) { + RelativePatcherTargetProvider* provider) { class RelativePatcherNone FINAL : public RelativePatcher { public: RelativePatcherNone() { } @@ -93,8 +92,7 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create( }; UNUSED(features); - UNUSED(thunk_provider); - UNUSED(target_provider); + UNUSED(provider); switch (instruction_set) { #ifdef ART_ENABLE_CODEGEN_x86 case InstructionSet::kX86: @@ -108,15 +106,12 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create( case InstructionSet::kArm: // Fall through: we generate Thumb2 code for "arm". case InstructionSet::kThumb2: - return std::unique_ptr<RelativePatcher>( - new Thumb2RelativePatcher(thunk_provider, target_provider)); + return std::unique_ptr<RelativePatcher>(new Thumb2RelativePatcher(provider)); #endif #ifdef ART_ENABLE_CODEGEN_arm64 case InstructionSet::kArm64: return std::unique_ptr<RelativePatcher>( - new Arm64RelativePatcher(thunk_provider, - target_provider, - features->AsArm64InstructionSetFeatures())); + new Arm64RelativePatcher(provider, features->AsArm64InstructionSetFeatures())); #endif #ifdef ART_ENABLE_CODEGEN_mips case InstructionSet::kMips: diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h index 06c7e70d23..b58e3dffbd 100644 --- a/compiler/linker/relative_patcher.h +++ b/compiler/linker/relative_patcher.h @@ -39,27 +39,6 @@ class LinkerPatch; class OutputStream; /** - * @class RelativePatcherThunkProvider - * @brief Interface for providing method offsets for relative call targets. - */ -class RelativePatcherThunkProvider { - public: - /** - * Get the code and debug name of a thunk needed by the given linker patch. - * - * @param patch The patch for which we need to retrieve the thunk code. - * @param code A variable to receive the code of the thunk. This code must not be empty. - * @param debug_name A variable to receive the debug name of the thunk. - */ - virtual void GetThunkCode(const LinkerPatch& patch, - /*out*/ ArrayRef<const uint8_t>* code, - /*out*/ std::string* debug_name) = 0; - - protected: - virtual ~RelativePatcherThunkProvider() { } -}; - -/** * @class RelativePatcherTargetProvider * @brief Interface for providing method offsets for relative call targets. */ @@ -91,10 +70,8 @@ class RelativePatcherTargetProvider { class RelativePatcher { public: static std::unique_ptr<RelativePatcher> Create( - InstructionSet instruction_set, - const InstructionSetFeatures* features, - RelativePatcherThunkProvider* thunk_provider, - RelativePatcherTargetProvider* target_provider); + InstructionSet instruction_set, const InstructionSetFeatures* features, + RelativePatcherTargetProvider* provider); virtual ~RelativePatcher() { } diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h index af8dc4dbc9..d21f2795b9 100644 --- a/compiler/linker/relative_patcher_test.h +++ b/compiler/linker/relative_patcher_test.h @@ -58,10 +58,7 @@ class RelativePatcherTest : public testing::Test { instruction_set_(instruction_set), features_(InstructionSetFeatures::FromVariant(instruction_set, variant, &error_msg_)), method_offset_map_(), - patcher_(RelativePatcher::Create(instruction_set, - features_.get(), - &thunk_provider_, - &method_offset_map_)), + patcher_(RelativePatcher::Create(instruction_set, features_.get(), &method_offset_map_)), bss_begin_(0u), compiled_method_refs_(), compiled_methods_(), @@ -251,72 +248,6 @@ class RelativePatcherTest : public testing::Test { LOG(ERROR) << " " << diff_indicator_str; } - class ThunkProvider : public RelativePatcherThunkProvider { - public: - ThunkProvider() {} - - void SetThunkCode(const LinkerPatch& patch, - ArrayRef<const uint8_t> code, - const std::string& debug_name) { - thunk_map_.emplace(ThunkKey(patch), ThunkValue(code, debug_name)); - } - - void GetThunkCode(const LinkerPatch& patch, - /*out*/ ArrayRef<const uint8_t>* code, - /*out*/ std::string* debug_name) OVERRIDE { - auto it = thunk_map_.find(ThunkKey(patch)); - CHECK(it != thunk_map_.end()); - const ThunkValue& value = it->second; - CHECK(code != nullptr); - *code = value.GetCode(); - CHECK(debug_name != nullptr); - *debug_name = value.GetDebugName(); - } - - private: - class ThunkKey { - public: - explicit ThunkKey(const LinkerPatch& patch) - : type_(patch.GetType()), - custom_value1_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch - ? patch.GetBakerCustomValue1() : 0u), - custom_value2_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch - ? patch.GetBakerCustomValue2() : 0u) { - CHECK(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || - patch.GetType() == LinkerPatch::Type::kCallRelative); - } - - bool operator<(const ThunkKey& other) const { - if (custom_value1_ != other.custom_value1_) { - return custom_value1_ < other.custom_value1_; - } - if (custom_value2_ != other.custom_value2_) { - return custom_value2_ < other.custom_value2_; - } - return type_ < other.type_; - } - - private: - const LinkerPatch::Type type_; - const uint32_t custom_value1_; - const uint32_t custom_value2_; - }; - - class ThunkValue { - public: - ThunkValue(ArrayRef<const uint8_t> code, const std::string& debug_name) - : code_(code.begin(), code.end()), debug_name_(debug_name) {} - ArrayRef<const uint8_t> GetCode() const { return ArrayRef<const uint8_t>(code_); } - const std::string& GetDebugName() const { return debug_name_; } - - private: - const std::vector<uint8_t> code_; - const std::string debug_name_; - }; - - std::map<ThunkKey, ThunkValue> thunk_map_; - }; - // Map method reference to assinged offset. // Wrap the map in a class implementing RelativePatcherTargetProvider. class MethodOffsetMap FINAL : public RelativePatcherTargetProvider { @@ -341,7 +272,6 @@ class RelativePatcherTest : public testing::Test { std::string error_msg_; InstructionSet instruction_set_; std::unique_ptr<const InstructionSetFeatures> features_; - ThunkProvider thunk_provider_; MethodOffsetMap method_offset_map_; std::unique_ptr<RelativePatcher> patcher_; uint32_t bss_begin_; |