summaryrefslogtreecommitdiff
path: root/compiler/linker
diff options
context:
space:
mode:
author Vladimir Marko <vmarko@google.com> 2017-11-23 16:05:19 +0000
committer Vladimir Marko <vmarko@google.com> 2018-04-04 10:34:36 +0100
commitc9dd2207dfdab42586b1d6a5e7f11cf2fcea3a7a (patch)
tree879df31fd10658093b8931117ee617064ce82519 /compiler/linker
parent30a2d9c61da75359dee4ce90236d19fc6341b07a (diff)
Compile link-time thunks in codegen.
Prepare for experimenting with Baker read barrier marking introspection entrypoints for JIT. Test: m test-art-host-gtest Test: Compare compiled boot*.oat before and after (no diff). Test: Pixel 2 XL boots. Bug: 36141117 Change-Id: Idb413a31b158db4bf89a8707ea46dd167a06f110
Diffstat (limited to 'compiler/linker')
-rw-r--r--compiler/linker/arm/relative_patcher_arm_base.cc44
-rw-r--r--compiler/linker/arm/relative_patcher_arm_base.h9
-rw-r--r--compiler/linker/arm/relative_patcher_thumb2.cc305
-rw-r--r--compiler/linker/arm/relative_patcher_thumb2.h78
-rw-r--r--compiler/linker/arm/relative_patcher_thumb2_test.cc96
-rw-r--r--compiler/linker/arm64/relative_patcher_arm64.cc253
-rw-r--r--compiler/linker/arm64/relative_patcher_arm64.h54
-rw-r--r--compiler/linker/arm64/relative_patcher_arm64_test.cc87
-rw-r--r--compiler/linker/linker_patch.h2
-rw-r--r--compiler/linker/relative_patcher.cc13
-rw-r--r--compiler/linker/relative_patcher.h27
-rw-r--r--compiler/linker/relative_patcher_test.h72
12 files changed, 284 insertions, 756 deletions
diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc
index 6e0286afac..7cb8ae55c5 100644
--- a/compiler/linker/arm/relative_patcher_arm_base.cc
+++ b/compiler/linker/arm/relative_patcher_arm_base.cc
@@ -30,8 +30,9 @@ namespace linker {
class ArmBaseRelativePatcher::ThunkData {
public:
- ThunkData(std::vector<uint8_t> code, uint32_t max_next_offset)
- : code_(std::move(code)),
+ ThunkData(ArrayRef<const uint8_t> code, const std::string& debug_name, uint32_t max_next_offset)
+ : code_(code),
+ debug_name_(debug_name),
offsets_(),
max_next_offset_(max_next_offset),
pending_offset_(0u) {
@@ -45,7 +46,11 @@ class ArmBaseRelativePatcher::ThunkData {
}
ArrayRef<const uint8_t> GetCode() const {
- return ArrayRef<const uint8_t>(code_);
+ return code_;
+ }
+
+ const std::string& GetDebugName() const {
+ return debug_name_;
}
bool NeedsNextThunk() const {
@@ -142,10 +147,11 @@ class ArmBaseRelativePatcher::ThunkData {
}
private:
- std::vector<uint8_t> code_; // The code of the thunk.
- std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written.
- uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed.
- uint32_t pending_offset_; // The index of the next offset to write.
+ const ArrayRef<const uint8_t> code_; // The code of the thunk.
+ const std::string debug_name_; // The debug name of the thunk.
+ std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written.
+ uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed.
+ uint32_t pending_offset_; // The index of the next offset to write.
};
class ArmBaseRelativePatcher::PendingThunkComparator {
@@ -239,14 +245,13 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn
std::vector<debug::MethodDebugInfo> result;
result.reserve(number_of_thunks);
for (auto&& entry : thunks_) {
- const ThunkKey& key = entry.first;
const ThunkData& data = entry.second;
size_t start = data.IndexOfFirstThunkAtOrAfter(executable_offset);
if (start == data.NumberOfThunks()) {
continue;
}
// Get the base name to use for the first occurrence of the thunk.
- std::string base_name = GetThunkDebugName(key);
+ std::string base_name = data.GetDebugName();
for (size_t i = start, num = data.NumberOfThunks(); i != num; ++i) {
debug::MethodDebugInfo info = {};
if (i == 0u) {
@@ -267,9 +272,11 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn
return result;
}
-ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider,
+ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
InstructionSet instruction_set)
- : provider_(provider),
+ : thunk_provider_(thunk_provider),
+ target_provider_(target_provider),
instruction_set_(instruction_set),
thunks_(),
unprocessed_method_call_patches_(),
@@ -398,7 +405,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho
unprocessed_method_call_patches_.emplace_back(patch_offset, patch.TargetMethod());
if (method_call_thunk_ == nullptr) {
uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.Put(key, ThunkData(CompileThunk(key), max_next_offset));
+ auto it = thunks_.Put(key, ThunkDataForPatch(patch, max_next_offset));
method_call_thunk_ = &it->second;
AddUnreservedThunk(method_call_thunk_);
} else {
@@ -409,7 +416,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho
auto lb = thunks_.lower_bound(key);
if (lb == thunks_.end() || thunks_.key_comp()(key, lb->first)) {
uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.PutBefore(lb, key, ThunkData(CompileThunk(key), max_next_offset));
+ auto it = thunks_.PutBefore(lb, key, ThunkDataForPatch(patch, max_next_offset));
AddUnreservedThunk(&it->second);
} else {
old_data = &lb->second;
@@ -477,7 +484,7 @@ void ArmBaseRelativePatcher::ResolveMethodCalls(uint32_t quick_code_offset,
break;
}
} else {
- auto result = provider_->FindMethodOffset(target_method);
+ auto result = target_provider_->FindMethodOffset(target_method);
if (!result.first) {
break;
}
@@ -518,5 +525,14 @@ inline uint32_t ArmBaseRelativePatcher::CalculateMaxNextOffset(uint32_t patch_of
GetInstructionSetAlignment(instruction_set_));
}
+inline ArmBaseRelativePatcher::ThunkData ArmBaseRelativePatcher::ThunkDataForPatch(
+ const LinkerPatch& patch, uint32_t max_next_offset) {
+ ArrayRef<const uint8_t> code;
+ std::string debug_name;
+ thunk_provider_->GetThunkCode(patch, &code, &debug_name);
+ DCHECK(!code.empty());
+ return ThunkData(code, debug_name, max_next_offset);
+}
+
} // namespace linker
} // namespace art
diff --git a/compiler/linker/arm/relative_patcher_arm_base.h b/compiler/linker/arm/relative_patcher_arm_base.h
index ee09bf96b3..963d6690b0 100644
--- a/compiler/linker/arm/relative_patcher_arm_base.h
+++ b/compiler/linker/arm/relative_patcher_arm_base.h
@@ -37,7 +37,8 @@ class ArmBaseRelativePatcher : public RelativePatcher {
std::vector<debug::MethodDebugInfo> GenerateThunkDebugInfo(uint32_t executable_offset) OVERRIDE;
protected:
- ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider,
+ ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
InstructionSet instruction_set);
~ArmBaseRelativePatcher();
@@ -94,8 +95,6 @@ class ArmBaseRelativePatcher : public RelativePatcher {
uint32_t CalculateMethodCallDisplacement(uint32_t patch_offset,
uint32_t target_offset);
- virtual std::vector<uint8_t> CompileThunk(const ThunkKey& key) = 0;
- virtual std::string GetThunkDebugName(const ThunkKey& key) = 0;
virtual uint32_t MaxPositiveDisplacement(const ThunkKey& key) = 0;
virtual uint32_t MaxNegativeDisplacement(const ThunkKey& key) = 0;
@@ -108,8 +107,10 @@ class ArmBaseRelativePatcher : public RelativePatcher {
void ResolveMethodCalls(uint32_t quick_code_offset, MethodReference method_ref);
uint32_t CalculateMaxNextOffset(uint32_t patch_offset, const ThunkKey& key);
+ ThunkData ThunkDataForPatch(const LinkerPatch& patch, uint32_t max_next_offset);
- RelativePatcherTargetProvider* const provider_;
+ RelativePatcherThunkProvider* const thunk_provider_;
+ RelativePatcherTargetProvider* const target_provider_;
const InstructionSet instruction_set_;
// The data for all thunks.
diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc
index 78755176e4..7400d11c31 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2.cc
@@ -48,8 +48,9 @@ constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 24) - kPcDisplace
constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 2u + kPcDisplacement;
constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20) - kPcDisplacement;
-Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherTargetProvider* provider)
- : ArmBaseRelativePatcher(provider, InstructionSet::kThumb2) {
+Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider)
+ : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kThumb2) {
}
void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code,
@@ -110,62 +111,6 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co
uint32_t insn = GetInsn32(code, literal_offset);
DCHECK_EQ(insn, 0xf0408000); // BNE +0 (unpatched)
ThunkKey key = GetBakerThunkKey(patch);
- if (kIsDebugBuild) {
- const uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- // Check that the next instruction matches the expected LDR.
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- if (width == BakerReadBarrierWidth::kWide) {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn32(code, literal_offset + 4u);
- // LDR (immediate), encoding T3, with correct base_reg.
- CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16));
- } else {
- DCHECK_GE(code->size() - literal_offset, 6u);
- uint32_t next_insn = GetInsn16(code, literal_offset + 4u);
- // LDR (immediate), encoding T1, with correct base_reg.
- CheckValidReg(next_insn & 0x7u); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3));
- }
- break;
- }
- case BakerReadBarrierKind::kArray: {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn32(code, literal_offset + 4u);
- // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]).
- CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16));
- CheckValidReg(next_insn & 0xf); // Check index register
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- if (width == BakerReadBarrierWidth::kWide) {
- DCHECK_GE(literal_offset, 4u);
- uint32_t prev_insn = GetInsn32(code, literal_offset - 4u);
- // LDR (immediate), encoding T3, with correct root_reg.
- const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12));
- } else {
- DCHECK_GE(literal_offset, 2u);
- uint32_t prev_insn = GetInsn16(code, literal_offset - 2u);
- // LDR (immediate), encoding T1, with correct root_reg.
- const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg);
- }
- break;
- }
- default:
- LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(key.GetType());
- UNREACHABLE();
- }
- }
uint32_t target_offset = GetThunkTargetOffset(key, patch_offset);
DCHECK_ALIGNED(target_offset, 4u);
uint32_t disp = target_offset - (patch_offset + kPcDisplacement);
@@ -178,250 +123,6 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co
SetInsn32(code, literal_offset, insn);
}
-#define __ assembler.GetVIXLAssembler()->
-
-static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler,
- vixl::aarch32::Register base_reg,
- vixl::aarch32::MemOperand& lock_word,
- vixl::aarch32::Label* slow_path,
- int32_t raw_ldr_offset) {
- using namespace vixl::aarch32; // NOLINT(build/namespaces)
- // Load the lock word containing the rb_state.
- __ Ldr(ip, lock_word);
- // Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted));
- __ B(ne, slow_path, /* is_far_target */ false);
- __ Add(lr, lr, raw_ldr_offset);
- // Introduce a dependency on the lock_word including rb_state,
- // to prevent load-load reordering, and without using
- // a memory barrier (which would be more expensive).
- __ Add(base_reg, base_reg, Operand(ip, LSR, 32));
- __ Bx(lr); // And return back to the function.
- // Note: The fake dependency is unnecessary for the slow path.
-}
-
-// Load the read barrier introspection entrypoint in register `entrypoint`
-static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler,
- vixl::aarch32::Register entrypoint) {
- using vixl::aarch32::MemOperand;
- using vixl::aarch32::ip;
- // Thread Register.
- const vixl::aarch32::Register tr = vixl::aarch32::r9;
-
- // The register where the read barrier introspection entrypoint is loaded
- // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4).
- DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister);
- // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection.
- DCHECK_EQ(ip.GetCode(), 12u);
- const int32_t entry_point_offset =
- Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode());
- __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
-}
-
-void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler,
- uint32_t encoded_data) {
- using namespace vixl::aarch32; // NOLINT(build/namespaces)
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- // Check if the holder is gray and, if not, add fake dependency to the base register
- // and return to the LDR instruction to load the reference. Otherwise, use introspection
- // to load the reference and call the entrypoint (in kBakerCcEntrypointRegister)
- // that performs further checks on the reference and marks it if needed.
- Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data));
- CheckValidReg(holder_reg.GetCode());
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip);
- // If base_reg differs from holder_reg, the offset was too large and we must have
- // emitted an explicit null check before the load. Otherwise, we need to null-check
- // the holder as we do not necessarily do that check before going to the thunk.
- vixl::aarch32::Label throw_npe;
- if (holder_reg.Is(base_reg)) {
- __ CompareAndBranchIfZero(holder_reg, &throw_npe, /* is_far_target */ false);
- }
- vixl::aarch32::Label slow_path;
- MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
- const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide)
- ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET
- : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET;
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
- __ Bind(&slow_path);
- const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
- raw_ldr_offset;
- Register ep_reg(kBakerCcEntrypointRegister);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
- if (width == BakerReadBarrierWidth::kWide) {
- MemOperand ldr_half_address(lr, ldr_offset + 2);
- __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12".
- __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12.
- __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference.
- } else {
- MemOperand ldr_address(lr, ldr_offset);
- __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1.
- __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint
- ep_reg, // for narrow LDR.
- Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET));
- __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4.
- __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference.
- }
- // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
- __ Bx(ep_reg); // Jump to the entrypoint.
- if (holder_reg.Is(base_reg)) {
- // Add null check slow path. The stack map is at the address pointed to by LR.
- __ Bind(&throw_npe);
- int32_t offset = GetThreadOffset<kArmPointerSize>(kQuickThrowNullPointer).Int32Value();
- __ Ldr(ip, MemOperand(/* Thread* */ vixl::aarch32::r9, offset));
- __ Bx(ip);
- }
- break;
- }
- case BakerReadBarrierKind::kArray: {
- Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip);
- vixl::aarch32::Label slow_path;
- int32_t data_offset =
- mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
- MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
- DCHECK_LT(lock_word.GetOffsetImmediate(), 0);
- const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET;
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
- __ Bind(&slow_path);
- const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
- raw_ldr_offset;
- MemOperand ldr_address(lr, ldr_offset + 2);
- __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm",
- // i.e. Rm+32 because the scale in imm2 is 2.
- Register ep_reg(kBakerCcEntrypointRegister);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
- __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create
- // a switch case target based on the index register.
- __ Mov(ip, base_reg); // Move the base register to ip0.
- __ Bx(ep_reg); // Jump to the entrypoint's array switch case.
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
- // and it does not have a forwarding address), call the correct introspection entrypoint;
- // otherwise return the reference (or the extracted forwarding address).
- // There is no gray bit check for GC roots.
- Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(root_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip);
- vixl::aarch32::Label return_label, not_marked, forwarding_address;
- __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false);
- MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value());
- __ Ldr(ip, lock_word);
- __ Tst(ip, LockWord::kMarkBitStateMaskShifted);
- __ B(eq, &not_marked);
- __ Bind(&return_label);
- __ Bx(lr);
- __ Bind(&not_marked);
- static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3,
- "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in "
- " the highest bits and the 'forwarding address' state to have all bits set");
- __ Cmp(ip, Operand(0xc0000000));
- __ B(hs, &forwarding_address);
- Register ep_reg(kBakerCcEntrypointRegister);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
- // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister
- // to art_quick_read_barrier_mark_introspection_gc_roots.
- int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide)
- ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET
- : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET;
- __ Add(ep_reg, ep_reg, Operand(entrypoint_offset));
- __ Mov(ip, root_reg);
- __ Bx(ep_reg);
- __ Bind(&forwarding_address);
- __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift);
- __ Bx(lr);
- break;
- }
- default:
- LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
- UNREACHABLE();
- }
-}
-
-std::vector<uint8_t> Thumb2RelativePatcher::CompileThunk(const ThunkKey& key) {
- ArenaPool pool;
- ArenaAllocator allocator(&pool);
- arm::ArmVIXLAssembler assembler(&allocator);
-
- switch (key.GetType()) {
- case ThunkType::kMethodCall:
- // The thunk just uses the entry point in the ArtMethod. This works even for calls
- // to the generic JNI and interpreter trampolines.
- assembler.LoadFromOffset(
- arm::kLoadWord,
- vixl::aarch32::pc,
- vixl::aarch32::r0,
- ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value());
- __ Bkpt(0);
- break;
- case ThunkType::kBakerReadBarrier:
- CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1());
- break;
- }
-
- assembler.FinalizeCode();
- std::vector<uint8_t> thunk_code(assembler.CodeSize());
- MemoryRegion code(thunk_code.data(), thunk_code.size());
- assembler.FinalizeInstructions(code);
- return thunk_code;
-}
-
-std::string Thumb2RelativePatcher::GetThunkDebugName(const ThunkKey& key) {
- switch (key.GetType()) {
- case ThunkType::kMethodCall:
- return "MethodCallThunk";
-
- case ThunkType::kBakerReadBarrier: {
- uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- std::ostringstream oss;
- oss << "BakerReadBarrierThunk";
- switch (kind) {
- case BakerReadBarrierKind::kField:
- oss << "Field";
- if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
- oss << "Wide";
- }
- oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
- << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
- break;
- case BakerReadBarrierKind::kArray:
- oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
- break;
- case BakerReadBarrierKind::kGcRoot:
- oss << "GcRoot";
- if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
- oss << "Wide";
- }
- oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- break;
- }
- return oss.str();
- }
- }
-}
-
-#undef __
-
uint32_t Thumb2RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) {
switch (key.GetType()) {
case ThunkType::kMethodCall:
diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h
index 68386c00f4..68610d69e1 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.h
+++ b/compiler/linker/arm/relative_patcher_thumb2.h
@@ -19,8 +19,6 @@
#include "arch/arm/registers_arm.h"
#include "base/array_ref.h"
-#include "base/bit_field.h"
-#include "base/bit_utils.h"
#include "linker/arm/relative_patcher_arm_base.h"
namespace art {
@@ -33,42 +31,8 @@ namespace linker {
class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher {
public:
- static constexpr uint32_t kBakerCcEntrypointRegister = 4u;
-
- static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
- uint32_t holder_reg,
- bool narrow) {
- CheckValidReg(base_reg);
- CheckValidReg(holder_reg);
- DCHECK(!narrow || base_reg < 8u) << base_reg;
- BakerReadBarrierWidth width =
- narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(holder_reg) |
- BakerReadBarrierWidthField::Encode(width);
- }
-
- static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
- CheckValidReg(base_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) |
- BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide);
- }
-
- static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
- CheckValidReg(root_reg);
- DCHECK(!narrow || root_reg < 8u) << root_reg;
- BakerReadBarrierWidth width =
- narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
- BakerReadBarrierFirstRegField::Encode(root_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) |
- BakerReadBarrierWidthField::Encode(width);
- }
-
- explicit Thumb2RelativePatcher(RelativePatcherTargetProvider* provider);
+ explicit Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider);
void PatchCall(std::vector<uint8_t>* code,
uint32_t literal_offset,
@@ -83,48 +47,10 @@ class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher {
uint32_t patch_offset) OVERRIDE;
protected:
- std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE;
- std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE;
uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE;
uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE;
private:
- static constexpr uint32_t kInvalidEncodedReg = /* pc is invalid */ 15u;
-
- enum class BakerReadBarrierKind : uint8_t {
- kField, // Field get or array get with constant offset (i.e. constant index).
- kArray, // Array get with index in register.
- kGcRoot, // GC root load.
- kLast = kGcRoot
- };
-
- enum class BakerReadBarrierWidth : uint8_t {
- kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled).
- kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled).
- kLast = kNarrow
- };
-
- static constexpr size_t kBitsForBakerReadBarrierKind =
- MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
- static constexpr size_t kBitsForRegister = 4u;
- using BakerReadBarrierKindField =
- BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
- using BakerReadBarrierFirstRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>;
- using BakerReadBarrierSecondRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>;
- static constexpr size_t kBitsForBakerReadBarrierWidth =
- MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast));
- using BakerReadBarrierWidthField = BitField<BakerReadBarrierWidth,
- kBitsForBakerReadBarrierKind + 2 * kBitsForRegister,
- kBitsForBakerReadBarrierWidth>;
-
- static void CheckValidReg(uint32_t reg) {
- DCHECK(reg < 12u && reg != kBakerCcEntrypointRegister) << reg;
- }
-
- void CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data);
-
void SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value);
static uint32_t GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset);
diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc
index 2c22a352c2..e7b11bd16b 100644
--- a/compiler/linker/arm/relative_patcher_thumb2_test.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc
@@ -16,12 +16,15 @@
#include "linker/arm/relative_patcher_thumb2.h"
+#include "arch/arm/instruction_set_features_arm.h"
#include "base/casts.h"
#include "linker/relative_patcher_test.h"
#include "lock_word.h"
#include "mirror/array-inl.h"
#include "mirror/object.h"
#include "oat_quick_method_header.h"
+#include "optimizing/code_generator_arm_vixl.h"
+#include "optimizing/optimizing_unit_test.h"
namespace art {
namespace linker {
@@ -189,9 +192,42 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest {
return result.second - 1 /* thumb mode */;
}
+ std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
+ /*out*/ std::string* debug_name = nullptr) {
+ OptimizingUnitTestHelper helper;
+ HGraph* graph = helper.CreateGraph();
+ std::string error_msg;
+ ArmFeaturesUniquePtr features =
+ ArmInstructionSetFeatures::FromVariant("default", &error_msg);
+ CompilerOptions options;
+ arm::CodeGeneratorARMVIXL codegen(graph, *features, options);
+ ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
+ codegen.EmitThunkCode(patch, &code, debug_name);
+ return std::vector<uint8_t>(code.begin(), code.end());
+ }
+
+ void AddCompiledMethod(
+ MethodReference method_ref,
+ const ArrayRef<const uint8_t>& code,
+ const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
+ RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
+
+ // Make sure the ThunkProvider has all the necessary thunks.
+ for (const LinkerPatch& patch : patches) {
+ if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == LinkerPatch::Type::kCallRelative) {
+ std::string debug_name;
+ std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
+ thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
+ }
+ }
+ }
+
std::vector<uint8_t> CompileMethodCallThunk() {
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey();
- return static_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
+ /* target_dex_file*/ nullptr,
+ /* target_method_idx */ 0u);
+ return CompileThunk(patch);
}
uint32_t MethodCallThunkSize() {
@@ -228,27 +264,38 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest {
void TestStringReference(uint32_t string_offset);
void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
+ static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
+ uint32_t holder_reg,
+ bool narrow) {
+ return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow);
+ }
+
+ static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
+ return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow);
+ }
+
std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg,
uint32_t holder_reg,
bool narrow) {
const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow));
+ return CompileThunk(patch);
}
uint32_t GetOutputInsn32(uint32_t offset) {
@@ -594,7 +641,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref
const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
base_reg, holder_reg, /* narrow */ false);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
@@ -696,7 +743,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t r
const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
base_reg, holder_reg, /* narrow */ true);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
@@ -809,7 +856,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) {
constexpr uint32_t kLiteralOffset1 = 6u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -877,7 +924,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) {
constexpr uint32_t kLiteralOffset1 = 4u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -907,7 +954,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast
constexpr uint32_t kLiteralOffset1 = 6u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -993,7 +1040,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerArray) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)),
+ kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1074,8 +1121,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset,
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
+ kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1134,8 +1180,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset,
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
+ kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1182,8 +1227,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) {
patches.reserve(num_patches);
const uint32_t ldr =
kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12);
- uint32_t encoded_data =
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
+ uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
for (size_t i = 0; i != num_patches; ++i) {
PushBackInsn(&code, ldr);
PushBackInsn(&code, kBneWPlus0);
@@ -1264,10 +1308,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) {
ldr1, kBneWPlus0, // First GC root LDR with read barrier.
ldr2, kBneWPlus0, // Second GC root LDR with read barrier.
});
- uint32_t encoded_data1 =
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
- uint32_t encoded_data2 =
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
+ uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
+ uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
const LinkerPatch last_method_patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc
index b268204b4a..135e39d100 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64.cc
@@ -82,9 +82,10 @@ inline uint32_t MaxExtraSpace(size_t num_adrp, size_t code_size) {
} // anonymous namespace
-Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider,
+Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
const Arm64InstructionSetFeatures* features)
- : ArmBaseRelativePatcher(provider, InstructionSet::kArm64),
+ : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kArm64),
fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()),
reserved_adrp_thunks_(0u),
processed_adrp_thunks_(0u) {
@@ -313,44 +314,6 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod
uint32_t insn = GetInsn(code, literal_offset);
DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched)
ThunkKey key = GetBakerThunkKey(patch);
- if (kIsDebugBuild) {
- const uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- // Check that the next instruction matches the expected LDR.
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn(code, literal_offset + 4u);
- // LDR (immediate) with correct base_reg.
- CheckValidReg(next_insn & 0x1fu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5));
- break;
- }
- case BakerReadBarrierKind::kArray: {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn(code, literal_offset + 4u);
- // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL),
- // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2].
- CheckValidReg(next_insn & 0x1fu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5));
- CheckValidReg((next_insn >> 16) & 0x1f); // Check index register
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- DCHECK_GE(literal_offset, 4u);
- uint32_t prev_insn = GetInsn(code, literal_offset - 4u);
- // LDR (immediate) with correct root_reg.
- const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg);
- break;
- }
- default:
- LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
- UNREACHABLE();
- }
- }
uint32_t target_offset = GetThunkTargetOffset(key, patch_offset);
DCHECK_ALIGNED(target_offset, 4u);
uint32_t disp = target_offset - patch_offset;
@@ -359,216 +322,6 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod
SetInsn(code, literal_offset, insn);
}
-#define __ assembler.GetVIXLAssembler()->
-
-static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler,
- vixl::aarch64::Register base_reg,
- vixl::aarch64::MemOperand& lock_word,
- vixl::aarch64::Label* slow_path) {
- using namespace vixl::aarch64; // NOLINT(build/namespaces)
- // Load the lock word containing the rb_state.
- __ Ldr(ip0.W(), lock_word);
- // Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path);
- static_assert(
- BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET,
- "Field and array LDR offsets must be the same to reuse the same code.");
- // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning).
- static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
- "Field LDR must be 1 instruction (4B) before the return address label; "
- " 2 instructions (8B) for heap poisoning.");
- __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
- // Introduce a dependency on the lock_word including rb_state,
- // to prevent load-load reordering, and without using
- // a memory barrier (which would be more expensive).
- __ Add(base_reg, base_reg, Operand(ip0, LSR, 32));
- __ Br(lr); // And return back to the function.
- // Note: The fake dependency is unnecessary for the slow path.
-}
-
-// Load the read barrier introspection entrypoint in register `entrypoint`.
-static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler,
- vixl::aarch64::Register entrypoint) {
- using vixl::aarch64::MemOperand;
- using vixl::aarch64::ip0;
- // Thread Register.
- const vixl::aarch64::Register tr = vixl::aarch64::x19;
-
- // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection.
- DCHECK_EQ(ip0.GetCode(), 16u);
- const int32_t entry_point_offset =
- Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode());
- __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
-}
-
-void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler,
- uint32_t encoded_data) {
- using namespace vixl::aarch64; // NOLINT(build/namespaces)
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- // Check if the holder is gray and, if not, add fake dependency to the base register
- // and return to the LDR instruction to load the reference. Otherwise, use introspection
- // to load the reference and call the entrypoint (in IP1) that performs further checks
- // on the reference and marks it if needed.
- auto base_reg =
- Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- auto holder_reg =
- Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data));
- CheckValidReg(holder_reg.GetCode());
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip0, ip1);
- // If base_reg differs from holder_reg, the offset was too large and we must have
- // emitted an explicit null check before the load. Otherwise, we need to null-check
- // the holder as we do not necessarily do that check before going to the thunk.
- vixl::aarch64::Label throw_npe;
- if (holder_reg.Is(base_reg)) {
- __ Cbz(holder_reg.W(), &throw_npe);
- }
- vixl::aarch64::Label slow_path;
- MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
- __ Bind(&slow_path);
- MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
- __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset.
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
- __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset.
- __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference.
- // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
- __ Br(ip1); // Jump to the entrypoint.
- if (holder_reg.Is(base_reg)) {
- // Add null check slow path. The stack map is at the address pointed to by LR.
- __ Bind(&throw_npe);
- int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value();
- __ Ldr(ip0, MemOperand(/* Thread* */ vixl::aarch64::x19, offset));
- __ Br(ip0);
- }
- break;
- }
- case BakerReadBarrierKind::kArray: {
- auto base_reg =
- Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip0, ip1);
- vixl::aarch64::Label slow_path;
- int32_t data_offset =
- mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
- MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
- DCHECK_LT(lock_word.GetOffset(), 0);
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
- __ Bind(&slow_path);
- MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET);
- __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset.
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
- __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set).
- __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create
- // a switch case target based on the index register.
- __ Mov(ip0, base_reg); // Move the base register to ip0.
- __ Br(ip1); // Jump to the entrypoint's array switch case.
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
- // and it does not have a forwarding address), call the correct introspection entrypoint;
- // otherwise return the reference (or the extracted forwarding address).
- // There is no gray bit check for GC roots.
- auto root_reg =
- Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(root_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip0, ip1);
- vixl::aarch64::Label return_label, not_marked, forwarding_address;
- __ Cbz(root_reg, &return_label);
- MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value());
- __ Ldr(ip0.W(), lock_word);
- __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, &not_marked);
- __ Bind(&return_label);
- __ Br(lr);
- __ Bind(&not_marked);
- __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1));
- __ B(&forwarding_address, mi);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
- // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to
- // art_quick_read_barrier_mark_introspection_gc_roots.
- __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET));
- __ Mov(ip0.W(), root_reg);
- __ Br(ip1);
- __ Bind(&forwarding_address);
- __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift);
- __ Br(lr);
- break;
- }
- default:
- LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
- UNREACHABLE();
- }
-}
-
-std::vector<uint8_t> Arm64RelativePatcher::CompileThunk(const ThunkKey& key) {
- ArenaPool pool;
- ArenaAllocator allocator(&pool);
- arm64::Arm64Assembler assembler(&allocator);
-
- switch (key.GetType()) {
- case ThunkType::kMethodCall: {
- // The thunk just uses the entry point in the ArtMethod. This works even for calls
- // to the generic JNI and interpreter trampolines.
- Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset(
- kArm64PointerSize).Int32Value());
- assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0));
- break;
- }
- case ThunkType::kBakerReadBarrier: {
- CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1());
- break;
- }
- }
-
- // Ensure we emit the literal pool.
- assembler.FinalizeCode();
- std::vector<uint8_t> thunk_code(assembler.CodeSize());
- MemoryRegion code(thunk_code.data(), thunk_code.size());
- assembler.FinalizeInstructions(code);
- return thunk_code;
-}
-
-std::string Arm64RelativePatcher::GetThunkDebugName(const ThunkKey& key) {
- switch (key.GetType()) {
- case ThunkType::kMethodCall:
- return "MethodCallThunk";
-
- case ThunkType::kBakerReadBarrier: {
- uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- std::ostringstream oss;
- oss << "BakerReadBarrierThunk";
- switch (kind) {
- case BakerReadBarrierKind::kField:
- oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
- << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
- break;
- case BakerReadBarrierKind::kArray:
- oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- break;
- case BakerReadBarrierKind::kGcRoot:
- oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- break;
- }
- return oss.str();
- }
- }
-}
-
-#undef __
-
uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) {
switch (key.GetType()) {
case ThunkType::kMethodCall:
diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h
index 8ba59976e7..9dc289da44 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.h
+++ b/compiler/linker/arm64/relative_patcher_arm64.h
@@ -18,8 +18,6 @@
#define ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_
#include "base/array_ref.h"
-#include "base/bit_field.h"
-#include "base/bit_utils.h"
#include "linker/arm/relative_patcher_arm_base.h"
namespace art {
@@ -32,29 +30,8 @@ namespace linker {
class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher {
public:
- static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
- CheckValidReg(base_reg);
- CheckValidReg(holder_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(holder_reg);
- }
-
- static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
- CheckValidReg(base_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg);
- }
-
- static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
- CheckValidReg(root_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
- BakerReadBarrierFirstRegField::Encode(root_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg);
- }
-
- Arm64RelativePatcher(RelativePatcherTargetProvider* provider,
+ Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
const Arm64InstructionSetFeatures* features);
uint32_t ReserveSpace(uint32_t offset,
@@ -75,37 +52,10 @@ class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher {
uint32_t patch_offset) OVERRIDE;
protected:
- std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE;
- std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE;
uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE;
uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE;
private:
- static constexpr uint32_t kInvalidEncodedReg = /* sp/zr is invalid */ 31u;
-
- enum class BakerReadBarrierKind : uint8_t {
- kField, // Field get or array get with constant offset (i.e. constant index).
- kArray, // Array get with index in register.
- kGcRoot, // GC root load.
- kLast = kGcRoot
- };
-
- static constexpr size_t kBitsForBakerReadBarrierKind =
- MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
- static constexpr size_t kBitsForRegister = 5u;
- using BakerReadBarrierKindField =
- BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
- using BakerReadBarrierFirstRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>;
- using BakerReadBarrierSecondRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>;
-
- static void CheckValidReg(uint32_t reg) {
- DCHECK(reg < 30u && reg != 16u && reg != 17u) << reg;
- }
-
- void CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data);
-
static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp);
static bool NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, uint32_t literal_offset,
diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc
index 05459a2a82..393733dd0c 100644
--- a/compiler/linker/arm64/relative_patcher_arm64_test.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc
@@ -16,12 +16,15 @@
#include "linker/arm64/relative_patcher_arm64.h"
+#include "arch/arm64/instruction_set_features_arm64.h"
#include "base/casts.h"
#include "linker/relative_patcher_test.h"
#include "lock_word.h"
#include "mirror/array-inl.h"
#include "mirror/object.h"
#include "oat_quick_method_header.h"
+#include "optimizing/code_generator_arm64.h"
+#include "optimizing/optimizing_unit_test.h"
namespace art {
namespace linker {
@@ -168,9 +171,42 @@ class Arm64RelativePatcherTest : public RelativePatcherTest {
return result.second;
}
+ std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
+ /*out*/ std::string* debug_name = nullptr) {
+ OptimizingUnitTestHelper helper;
+ HGraph* graph = helper.CreateGraph();
+ std::string error_msg;
+ Arm64FeaturesUniquePtr features =
+ Arm64InstructionSetFeatures::FromVariant("default", &error_msg);
+ CompilerOptions options;
+ arm64::CodeGeneratorARM64 codegen(graph, *features, options);
+ ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
+ codegen.EmitThunkCode(patch, &code, debug_name);
+ return std::vector<uint8_t>(code.begin(), code.end());
+ }
+
+ void AddCompiledMethod(
+ MethodReference method_ref,
+ const ArrayRef<const uint8_t>& code,
+ const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
+ RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
+
+ // Make sure the ThunkProvider has all the necessary thunks.
+ for (const LinkerPatch& patch : patches) {
+ if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == LinkerPatch::Type::kCallRelative) {
+ std::string debug_name;
+ std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
+ thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
+ }
+ }
+ }
+
std::vector<uint8_t> CompileMethodCallThunk() {
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey();
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
+ /* target_dex_file*/ nullptr,
+ /* target_method_idx */ 0u);
+ return CompileThunk(patch);
}
uint32_t MethodCallThunkSize() {
@@ -475,25 +511,34 @@ class Arm64RelativePatcherTest : public RelativePatcherTest {
TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
}
+ static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
+ return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierArrayData(base_reg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
+ return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierGcRootData(root_reg);
+ }
+
std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) {
const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg));
+ return CompileThunk(patch);
}
uint32_t GetOutputInsn(uint32_t offset) {
@@ -919,8 +964,7 @@ void Arm64RelativePatcherTest::TestBakerField(uint32_t offset, uint32_t ref_reg)
const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
};
@@ -1005,8 +1049,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddle) {
constexpr uint32_t kLiteralOffset1 = 4;
const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1066,8 +1109,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkBeforeFiller) {
constexpr uint32_t kLiteralOffset1 = 0;
const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1096,8 +1138,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddleUnreachableFr
constexpr uint32_t kLiteralOffset1 = 4;
const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1170,7 +1211,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerArray) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)),
+ kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1247,7 +1288,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerGcRoot) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)),
+ kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1343,8 +1384,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerAndMethodCallInteraction) {
kNopInsn, kNopInsn, // Padding before second GC root read barrier.
ldr2, kCbnzIP1Plus0Insn, // Second GC root LDR with read barrier.
});
- uint32_t encoded_data1 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
- uint32_t encoded_data2 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
+ uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
+ uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
const LinkerPatch last_method_patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
diff --git a/compiler/linker/linker_patch.h b/compiler/linker/linker_patch.h
index 710d8a690a..7b35fd9b0c 100644
--- a/compiler/linker/linker_patch.h
+++ b/compiler/linker/linker_patch.h
@@ -141,7 +141,7 @@ class LinkerPatch {
static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset,
uint32_t custom_value1 = 0u,
uint32_t custom_value2 = 0u) {
- LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, nullptr);
+ LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, /* target_dex_file */ nullptr);
patch.baker_custom_value1_ = custom_value1;
patch.baker_custom_value2_ = custom_value2;
return patch;
diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc
index 13877f8f12..b82d15230d 100644
--- a/compiler/linker/relative_patcher.cc
+++ b/compiler/linker/relative_patcher.cc
@@ -43,7 +43,8 @@ namespace linker {
std::unique_ptr<RelativePatcher> RelativePatcher::Create(
InstructionSet instruction_set,
const InstructionSetFeatures* features,
- RelativePatcherTargetProvider* provider) {
+ RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider) {
class RelativePatcherNone FINAL : public RelativePatcher {
public:
RelativePatcherNone() { }
@@ -92,7 +93,8 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create(
};
UNUSED(features);
- UNUSED(provider);
+ UNUSED(thunk_provider);
+ UNUSED(target_provider);
switch (instruction_set) {
#ifdef ART_ENABLE_CODEGEN_x86
case InstructionSet::kX86:
@@ -106,12 +108,15 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create(
case InstructionSet::kArm:
// Fall through: we generate Thumb2 code for "arm".
case InstructionSet::kThumb2:
- return std::unique_ptr<RelativePatcher>(new Thumb2RelativePatcher(provider));
+ return std::unique_ptr<RelativePatcher>(
+ new Thumb2RelativePatcher(thunk_provider, target_provider));
#endif
#ifdef ART_ENABLE_CODEGEN_arm64
case InstructionSet::kArm64:
return std::unique_ptr<RelativePatcher>(
- new Arm64RelativePatcher(provider, features->AsArm64InstructionSetFeatures()));
+ new Arm64RelativePatcher(thunk_provider,
+ target_provider,
+ features->AsArm64InstructionSetFeatures()));
#endif
#ifdef ART_ENABLE_CODEGEN_mips
case InstructionSet::kMips:
diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h
index b58e3dffbd..06c7e70d23 100644
--- a/compiler/linker/relative_patcher.h
+++ b/compiler/linker/relative_patcher.h
@@ -39,6 +39,27 @@ class LinkerPatch;
class OutputStream;
/**
+ * @class RelativePatcherThunkProvider
+ * @brief Interface for providing method offsets for relative call targets.
+ */
+class RelativePatcherThunkProvider {
+ public:
+ /**
+ * Get the code and debug name of a thunk needed by the given linker patch.
+ *
+ * @param patch The patch for which we need to retrieve the thunk code.
+ * @param code A variable to receive the code of the thunk. This code must not be empty.
+ * @param debug_name A variable to receive the debug name of the thunk.
+ */
+ virtual void GetThunkCode(const LinkerPatch& patch,
+ /*out*/ ArrayRef<const uint8_t>* code,
+ /*out*/ std::string* debug_name) = 0;
+
+ protected:
+ virtual ~RelativePatcherThunkProvider() { }
+};
+
+/**
* @class RelativePatcherTargetProvider
* @brief Interface for providing method offsets for relative call targets.
*/
@@ -70,8 +91,10 @@ class RelativePatcherTargetProvider {
class RelativePatcher {
public:
static std::unique_ptr<RelativePatcher> Create(
- InstructionSet instruction_set, const InstructionSetFeatures* features,
- RelativePatcherTargetProvider* provider);
+ InstructionSet instruction_set,
+ const InstructionSetFeatures* features,
+ RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider);
virtual ~RelativePatcher() { }
diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h
index d21f2795b9..af8dc4dbc9 100644
--- a/compiler/linker/relative_patcher_test.h
+++ b/compiler/linker/relative_patcher_test.h
@@ -58,7 +58,10 @@ class RelativePatcherTest : public testing::Test {
instruction_set_(instruction_set),
features_(InstructionSetFeatures::FromVariant(instruction_set, variant, &error_msg_)),
method_offset_map_(),
- patcher_(RelativePatcher::Create(instruction_set, features_.get(), &method_offset_map_)),
+ patcher_(RelativePatcher::Create(instruction_set,
+ features_.get(),
+ &thunk_provider_,
+ &method_offset_map_)),
bss_begin_(0u),
compiled_method_refs_(),
compiled_methods_(),
@@ -248,6 +251,72 @@ class RelativePatcherTest : public testing::Test {
LOG(ERROR) << " " << diff_indicator_str;
}
+ class ThunkProvider : public RelativePatcherThunkProvider {
+ public:
+ ThunkProvider() {}
+
+ void SetThunkCode(const LinkerPatch& patch,
+ ArrayRef<const uint8_t> code,
+ const std::string& debug_name) {
+ thunk_map_.emplace(ThunkKey(patch), ThunkValue(code, debug_name));
+ }
+
+ void GetThunkCode(const LinkerPatch& patch,
+ /*out*/ ArrayRef<const uint8_t>* code,
+ /*out*/ std::string* debug_name) OVERRIDE {
+ auto it = thunk_map_.find(ThunkKey(patch));
+ CHECK(it != thunk_map_.end());
+ const ThunkValue& value = it->second;
+ CHECK(code != nullptr);
+ *code = value.GetCode();
+ CHECK(debug_name != nullptr);
+ *debug_name = value.GetDebugName();
+ }
+
+ private:
+ class ThunkKey {
+ public:
+ explicit ThunkKey(const LinkerPatch& patch)
+ : type_(patch.GetType()),
+ custom_value1_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch
+ ? patch.GetBakerCustomValue1() : 0u),
+ custom_value2_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch
+ ? patch.GetBakerCustomValue2() : 0u) {
+ CHECK(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == LinkerPatch::Type::kCallRelative);
+ }
+
+ bool operator<(const ThunkKey& other) const {
+ if (custom_value1_ != other.custom_value1_) {
+ return custom_value1_ < other.custom_value1_;
+ }
+ if (custom_value2_ != other.custom_value2_) {
+ return custom_value2_ < other.custom_value2_;
+ }
+ return type_ < other.type_;
+ }
+
+ private:
+ const LinkerPatch::Type type_;
+ const uint32_t custom_value1_;
+ const uint32_t custom_value2_;
+ };
+
+ class ThunkValue {
+ public:
+ ThunkValue(ArrayRef<const uint8_t> code, const std::string& debug_name)
+ : code_(code.begin(), code.end()), debug_name_(debug_name) {}
+ ArrayRef<const uint8_t> GetCode() const { return ArrayRef<const uint8_t>(code_); }
+ const std::string& GetDebugName() const { return debug_name_; }
+
+ private:
+ const std::vector<uint8_t> code_;
+ const std::string debug_name_;
+ };
+
+ std::map<ThunkKey, ThunkValue> thunk_map_;
+ };
+
// Map method reference to assinged offset.
// Wrap the map in a class implementing RelativePatcherTargetProvider.
class MethodOffsetMap FINAL : public RelativePatcherTargetProvider {
@@ -272,6 +341,7 @@ class RelativePatcherTest : public testing::Test {
std::string error_msg_;
InstructionSet instruction_set_;
std::unique_ptr<const InstructionSetFeatures> features_;
+ ThunkProvider thunk_provider_;
MethodOffsetMap method_offset_map_;
std::unique_ptr<RelativePatcher> patcher_;
uint32_t bss_begin_;