summaryrefslogtreecommitdiff
path: root/compiler/linker
diff options
context:
space:
mode:
author Vladimir Marko <vmarko@google.com> 2018-04-04 17:23:28 +0000
committer Vladimir Marko <vmarko@google.com> 2018-04-04 17:23:28 +0000
commit5806a9ec99b5494b511e84c74f494f0b3a8ebec5 (patch)
treebb50d00ff0890c2e10f351f462b47b56b01e78ea /compiler/linker
parentc9dd2207dfdab42586b1d6a5e7f11cf2fcea3a7a (diff)
Revert "Compile link-time thunks in codegen."
Reason for revert: This caused clang linker crash in several branches. Bug: 77581732 This reverts commit c9dd2207dfdab42586b1d6a5e7f11cf2fcea3a7a. Change-Id: I1923809083cf41c4f19e3e60df03ae80517aaedb
Diffstat (limited to 'compiler/linker')
-rw-r--r--compiler/linker/arm/relative_patcher_arm_base.cc44
-rw-r--r--compiler/linker/arm/relative_patcher_arm_base.h9
-rw-r--r--compiler/linker/arm/relative_patcher_thumb2.cc305
-rw-r--r--compiler/linker/arm/relative_patcher_thumb2.h78
-rw-r--r--compiler/linker/arm/relative_patcher_thumb2_test.cc96
-rw-r--r--compiler/linker/arm64/relative_patcher_arm64.cc253
-rw-r--r--compiler/linker/arm64/relative_patcher_arm64.h54
-rw-r--r--compiler/linker/arm64/relative_patcher_arm64_test.cc87
-rw-r--r--compiler/linker/linker_patch.h2
-rw-r--r--compiler/linker/relative_patcher.cc13
-rw-r--r--compiler/linker/relative_patcher.h27
-rw-r--r--compiler/linker/relative_patcher_test.h72
12 files changed, 756 insertions, 284 deletions
diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc
index 7cb8ae55c5..6e0286afac 100644
--- a/compiler/linker/arm/relative_patcher_arm_base.cc
+++ b/compiler/linker/arm/relative_patcher_arm_base.cc
@@ -30,9 +30,8 @@ namespace linker {
class ArmBaseRelativePatcher::ThunkData {
public:
- ThunkData(ArrayRef<const uint8_t> code, const std::string& debug_name, uint32_t max_next_offset)
- : code_(code),
- debug_name_(debug_name),
+ ThunkData(std::vector<uint8_t> code, uint32_t max_next_offset)
+ : code_(std::move(code)),
offsets_(),
max_next_offset_(max_next_offset),
pending_offset_(0u) {
@@ -46,11 +45,7 @@ class ArmBaseRelativePatcher::ThunkData {
}
ArrayRef<const uint8_t> GetCode() const {
- return code_;
- }
-
- const std::string& GetDebugName() const {
- return debug_name_;
+ return ArrayRef<const uint8_t>(code_);
}
bool NeedsNextThunk() const {
@@ -147,11 +142,10 @@ class ArmBaseRelativePatcher::ThunkData {
}
private:
- const ArrayRef<const uint8_t> code_; // The code of the thunk.
- const std::string debug_name_; // The debug name of the thunk.
- std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written.
- uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed.
- uint32_t pending_offset_; // The index of the next offset to write.
+ std::vector<uint8_t> code_; // The code of the thunk.
+ std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written.
+ uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed.
+ uint32_t pending_offset_; // The index of the next offset to write.
};
class ArmBaseRelativePatcher::PendingThunkComparator {
@@ -245,13 +239,14 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn
std::vector<debug::MethodDebugInfo> result;
result.reserve(number_of_thunks);
for (auto&& entry : thunks_) {
+ const ThunkKey& key = entry.first;
const ThunkData& data = entry.second;
size_t start = data.IndexOfFirstThunkAtOrAfter(executable_offset);
if (start == data.NumberOfThunks()) {
continue;
}
// Get the base name to use for the first occurrence of the thunk.
- std::string base_name = data.GetDebugName();
+ std::string base_name = GetThunkDebugName(key);
for (size_t i = start, num = data.NumberOfThunks(); i != num; ++i) {
debug::MethodDebugInfo info = {};
if (i == 0u) {
@@ -272,11 +267,9 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn
return result;
}
-ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider,
+ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider,
InstructionSet instruction_set)
- : thunk_provider_(thunk_provider),
- target_provider_(target_provider),
+ : provider_(provider),
instruction_set_(instruction_set),
thunks_(),
unprocessed_method_call_patches_(),
@@ -405,7 +398,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho
unprocessed_method_call_patches_.emplace_back(patch_offset, patch.TargetMethod());
if (method_call_thunk_ == nullptr) {
uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.Put(key, ThunkDataForPatch(patch, max_next_offset));
+ auto it = thunks_.Put(key, ThunkData(CompileThunk(key), max_next_offset));
method_call_thunk_ = &it->second;
AddUnreservedThunk(method_call_thunk_);
} else {
@@ -416,7 +409,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho
auto lb = thunks_.lower_bound(key);
if (lb == thunks_.end() || thunks_.key_comp()(key, lb->first)) {
uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.PutBefore(lb, key, ThunkDataForPatch(patch, max_next_offset));
+ auto it = thunks_.PutBefore(lb, key, ThunkData(CompileThunk(key), max_next_offset));
AddUnreservedThunk(&it->second);
} else {
old_data = &lb->second;
@@ -484,7 +477,7 @@ void ArmBaseRelativePatcher::ResolveMethodCalls(uint32_t quick_code_offset,
break;
}
} else {
- auto result = target_provider_->FindMethodOffset(target_method);
+ auto result = provider_->FindMethodOffset(target_method);
if (!result.first) {
break;
}
@@ -525,14 +518,5 @@ inline uint32_t ArmBaseRelativePatcher::CalculateMaxNextOffset(uint32_t patch_of
GetInstructionSetAlignment(instruction_set_));
}
-inline ArmBaseRelativePatcher::ThunkData ArmBaseRelativePatcher::ThunkDataForPatch(
- const LinkerPatch& patch, uint32_t max_next_offset) {
- ArrayRef<const uint8_t> code;
- std::string debug_name;
- thunk_provider_->GetThunkCode(patch, &code, &debug_name);
- DCHECK(!code.empty());
- return ThunkData(code, debug_name, max_next_offset);
-}
-
} // namespace linker
} // namespace art
diff --git a/compiler/linker/arm/relative_patcher_arm_base.h b/compiler/linker/arm/relative_patcher_arm_base.h
index 963d6690b0..ee09bf96b3 100644
--- a/compiler/linker/arm/relative_patcher_arm_base.h
+++ b/compiler/linker/arm/relative_patcher_arm_base.h
@@ -37,8 +37,7 @@ class ArmBaseRelativePatcher : public RelativePatcher {
std::vector<debug::MethodDebugInfo> GenerateThunkDebugInfo(uint32_t executable_offset) OVERRIDE;
protected:
- ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider,
+ ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider,
InstructionSet instruction_set);
~ArmBaseRelativePatcher();
@@ -95,6 +94,8 @@ class ArmBaseRelativePatcher : public RelativePatcher {
uint32_t CalculateMethodCallDisplacement(uint32_t patch_offset,
uint32_t target_offset);
+ virtual std::vector<uint8_t> CompileThunk(const ThunkKey& key) = 0;
+ virtual std::string GetThunkDebugName(const ThunkKey& key) = 0;
virtual uint32_t MaxPositiveDisplacement(const ThunkKey& key) = 0;
virtual uint32_t MaxNegativeDisplacement(const ThunkKey& key) = 0;
@@ -107,10 +108,8 @@ class ArmBaseRelativePatcher : public RelativePatcher {
void ResolveMethodCalls(uint32_t quick_code_offset, MethodReference method_ref);
uint32_t CalculateMaxNextOffset(uint32_t patch_offset, const ThunkKey& key);
- ThunkData ThunkDataForPatch(const LinkerPatch& patch, uint32_t max_next_offset);
- RelativePatcherThunkProvider* const thunk_provider_;
- RelativePatcherTargetProvider* const target_provider_;
+ RelativePatcherTargetProvider* const provider_;
const InstructionSet instruction_set_;
// The data for all thunks.
diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc
index 7400d11c31..78755176e4 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2.cc
@@ -48,9 +48,8 @@ constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 24) - kPcDisplace
constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 2u + kPcDisplacement;
constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20) - kPcDisplacement;
-Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider)
- : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kThumb2) {
+Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherTargetProvider* provider)
+ : ArmBaseRelativePatcher(provider, InstructionSet::kThumb2) {
}
void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code,
@@ -111,6 +110,62 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co
uint32_t insn = GetInsn32(code, literal_offset);
DCHECK_EQ(insn, 0xf0408000); // BNE +0 (unpatched)
ThunkKey key = GetBakerThunkKey(patch);
+ if (kIsDebugBuild) {
+ const uint32_t encoded_data = key.GetCustomValue1();
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ // Check that the next instruction matches the expected LDR.
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ if (width == BakerReadBarrierWidth::kWide) {
+ DCHECK_GE(code->size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn32(code, literal_offset + 4u);
+ // LDR (immediate), encoding T3, with correct base_reg.
+ CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16));
+ } else {
+ DCHECK_GE(code->size() - literal_offset, 6u);
+ uint32_t next_insn = GetInsn16(code, literal_offset + 4u);
+ // LDR (immediate), encoding T1, with correct base_reg.
+ CheckValidReg(next_insn & 0x7u); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3));
+ }
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ DCHECK_GE(code->size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn32(code, literal_offset + 4u);
+ // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]).
+ CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16));
+ CheckValidReg(next_insn & 0xf); // Check index register
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ if (width == BakerReadBarrierWidth::kWide) {
+ DCHECK_GE(literal_offset, 4u);
+ uint32_t prev_insn = GetInsn32(code, literal_offset - 4u);
+ // LDR (immediate), encoding T3, with correct root_reg.
+ const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12));
+ } else {
+ DCHECK_GE(literal_offset, 2u);
+ uint32_t prev_insn = GetInsn16(code, literal_offset - 2u);
+ // LDR (immediate), encoding T1, with correct root_reg.
+ const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg);
+ }
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(key.GetType());
+ UNREACHABLE();
+ }
+ }
uint32_t target_offset = GetThunkTargetOffset(key, patch_offset);
DCHECK_ALIGNED(target_offset, 4u);
uint32_t disp = target_offset - (patch_offset + kPcDisplacement);
@@ -123,6 +178,250 @@ void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* co
SetInsn32(code, literal_offset, insn);
}
+#define __ assembler.GetVIXLAssembler()->
+
+static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler,
+ vixl::aarch32::Register base_reg,
+ vixl::aarch32::MemOperand& lock_word,
+ vixl::aarch32::Label* slow_path,
+ int32_t raw_ldr_offset) {
+ using namespace vixl::aarch32; // NOLINT(build/namespaces)
+ // Load the lock word containing the rb_state.
+ __ Ldr(ip, lock_word);
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted));
+ __ B(ne, slow_path, /* is_far_target */ false);
+ __ Add(lr, lr, raw_ldr_offset);
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ __ Add(base_reg, base_reg, Operand(ip, LSR, 32));
+ __ Bx(lr); // And return back to the function.
+ // Note: The fake dependency is unnecessary for the slow path.
+}
+
+// Load the read barrier introspection entrypoint in register `entrypoint`
+static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler,
+ vixl::aarch32::Register entrypoint) {
+ using vixl::aarch32::MemOperand;
+ using vixl::aarch32::ip;
+ // Thread Register.
+ const vixl::aarch32::Register tr = vixl::aarch32::r9;
+
+ // The register where the read barrier introspection entrypoint is loaded
+ // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4).
+ DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister);
+ // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection.
+ DCHECK_EQ(ip.GetCode(), 12u);
+ const int32_t entry_point_offset =
+ Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode());
+ __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
+}
+
+void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler,
+ uint32_t encoded_data) {
+ using namespace vixl::aarch32; // NOLINT(build/namespaces)
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ // Check if the holder is gray and, if not, add fake dependency to the base register
+ // and return to the LDR instruction to load the reference. Otherwise, use introspection
+ // to load the reference and call the entrypoint (in kBakerCcEntrypointRegister)
+ // that performs further checks on the reference and marks it if needed.
+ Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data));
+ CheckValidReg(holder_reg.GetCode());
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip);
+ // If base_reg differs from holder_reg, the offset was too large and we must have
+ // emitted an explicit null check before the load. Otherwise, we need to null-check
+ // the holder as we do not necessarily do that check before going to the thunk.
+ vixl::aarch32::Label throw_npe;
+ if (holder_reg.Is(base_reg)) {
+ __ CompareAndBranchIfZero(holder_reg, &throw_npe, /* is_far_target */ false);
+ }
+ vixl::aarch32::Label slow_path;
+ MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
+ const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide)
+ ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET
+ : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET;
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
+ __ Bind(&slow_path);
+ const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
+ raw_ldr_offset;
+ Register ep_reg(kBakerCcEntrypointRegister);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
+ if (width == BakerReadBarrierWidth::kWide) {
+ MemOperand ldr_half_address(lr, ldr_offset + 2);
+ __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12".
+ __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12.
+ __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference.
+ } else {
+ MemOperand ldr_address(lr, ldr_offset);
+ __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1.
+ __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint
+ ep_reg, // for narrow LDR.
+ Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET));
+ __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4.
+ __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference.
+ }
+ // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
+ __ Bx(ep_reg); // Jump to the entrypoint.
+ if (holder_reg.Is(base_reg)) {
+ // Add null check slow path. The stack map is at the address pointed to by LR.
+ __ Bind(&throw_npe);
+ int32_t offset = GetThreadOffset<kArmPointerSize>(kQuickThrowNullPointer).Int32Value();
+ __ Ldr(ip, MemOperand(/* Thread* */ vixl::aarch32::r9, offset));
+ __ Bx(ip);
+ }
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip);
+ vixl::aarch32::Label slow_path;
+ int32_t data_offset =
+ mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
+ MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
+ DCHECK_LT(lock_word.GetOffsetImmediate(), 0);
+ const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET;
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
+ __ Bind(&slow_path);
+ const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
+ raw_ldr_offset;
+ MemOperand ldr_address(lr, ldr_offset + 2);
+ __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm",
+ // i.e. Rm+32 because the scale in imm2 is 2.
+ Register ep_reg(kBakerCcEntrypointRegister);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
+ __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create
+ // a switch case target based on the index register.
+ __ Mov(ip, base_reg); // Move the base register to ip0.
+ __ Bx(ep_reg); // Jump to the entrypoint's array switch case.
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
+ // and it does not have a forwarding address), call the correct introspection entrypoint;
+ // otherwise return the reference (or the extracted forwarding address).
+ // There is no gray bit check for GC roots.
+ Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(root_reg.GetCode());
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip);
+ vixl::aarch32::Label return_label, not_marked, forwarding_address;
+ __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false);
+ MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value());
+ __ Ldr(ip, lock_word);
+ __ Tst(ip, LockWord::kMarkBitStateMaskShifted);
+ __ B(eq, &not_marked);
+ __ Bind(&return_label);
+ __ Bx(lr);
+ __ Bind(&not_marked);
+ static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3,
+ "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in "
+ " the highest bits and the 'forwarding address' state to have all bits set");
+ __ Cmp(ip, Operand(0xc0000000));
+ __ B(hs, &forwarding_address);
+ Register ep_reg(kBakerCcEntrypointRegister);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
+ // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister
+ // to art_quick_read_barrier_mark_introspection_gc_roots.
+ int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide)
+ ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET
+ : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET;
+ __ Add(ep_reg, ep_reg, Operand(entrypoint_offset));
+ __ Mov(ip, root_reg);
+ __ Bx(ep_reg);
+ __ Bind(&forwarding_address);
+ __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift);
+ __ Bx(lr);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
+ UNREACHABLE();
+ }
+}
+
+std::vector<uint8_t> Thumb2RelativePatcher::CompileThunk(const ThunkKey& key) {
+ ArenaPool pool;
+ ArenaAllocator allocator(&pool);
+ arm::ArmVIXLAssembler assembler(&allocator);
+
+ switch (key.GetType()) {
+ case ThunkType::kMethodCall:
+ // The thunk just uses the entry point in the ArtMethod. This works even for calls
+ // to the generic JNI and interpreter trampolines.
+ assembler.LoadFromOffset(
+ arm::kLoadWord,
+ vixl::aarch32::pc,
+ vixl::aarch32::r0,
+ ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value());
+ __ Bkpt(0);
+ break;
+ case ThunkType::kBakerReadBarrier:
+ CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1());
+ break;
+ }
+
+ assembler.FinalizeCode();
+ std::vector<uint8_t> thunk_code(assembler.CodeSize());
+ MemoryRegion code(thunk_code.data(), thunk_code.size());
+ assembler.FinalizeInstructions(code);
+ return thunk_code;
+}
+
+std::string Thumb2RelativePatcher::GetThunkDebugName(const ThunkKey& key) {
+ switch (key.GetType()) {
+ case ThunkType::kMethodCall:
+ return "MethodCallThunk";
+
+ case ThunkType::kBakerReadBarrier: {
+ uint32_t encoded_data = key.GetCustomValue1();
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ std::ostringstream oss;
+ oss << "BakerReadBarrierThunk";
+ switch (kind) {
+ case BakerReadBarrierKind::kField:
+ oss << "Field";
+ if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
+ oss << "Wide";
+ }
+ oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
+ << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
+ break;
+ case BakerReadBarrierKind::kArray:
+ oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
+ break;
+ case BakerReadBarrierKind::kGcRoot:
+ oss << "GcRoot";
+ if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
+ oss << "Wide";
+ }
+ oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ break;
+ }
+ return oss.str();
+ }
+ }
+}
+
+#undef __
+
uint32_t Thumb2RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) {
switch (key.GetType()) {
case ThunkType::kMethodCall:
diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h
index 68610d69e1..68386c00f4 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.h
+++ b/compiler/linker/arm/relative_patcher_thumb2.h
@@ -19,6 +19,8 @@
#include "arch/arm/registers_arm.h"
#include "base/array_ref.h"
+#include "base/bit_field.h"
+#include "base/bit_utils.h"
#include "linker/arm/relative_patcher_arm_base.h"
namespace art {
@@ -31,8 +33,42 @@ namespace linker {
class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher {
public:
- explicit Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider);
+ static constexpr uint32_t kBakerCcEntrypointRegister = 4u;
+
+ static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
+ uint32_t holder_reg,
+ bool narrow) {
+ CheckValidReg(base_reg);
+ CheckValidReg(holder_reg);
+ DCHECK(!narrow || base_reg < 8u) << base_reg;
+ BakerReadBarrierWidth width =
+ narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(holder_reg) |
+ BakerReadBarrierWidthField::Encode(width);
+ }
+
+ static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ CheckValidReg(base_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) |
+ BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide);
+ }
+
+ static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
+ CheckValidReg(root_reg);
+ DCHECK(!narrow || root_reg < 8u) << root_reg;
+ BakerReadBarrierWidth width =
+ narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
+ BakerReadBarrierFirstRegField::Encode(root_reg) |
+ BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) |
+ BakerReadBarrierWidthField::Encode(width);
+ }
+
+ explicit Thumb2RelativePatcher(RelativePatcherTargetProvider* provider);
void PatchCall(std::vector<uint8_t>* code,
uint32_t literal_offset,
@@ -47,10 +83,48 @@ class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher {
uint32_t patch_offset) OVERRIDE;
protected:
+ std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE;
+ std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE;
uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE;
uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE;
private:
+ static constexpr uint32_t kInvalidEncodedReg = /* pc is invalid */ 15u;
+
+ enum class BakerReadBarrierKind : uint8_t {
+ kField, // Field get or array get with constant offset (i.e. constant index).
+ kArray, // Array get with index in register.
+ kGcRoot, // GC root load.
+ kLast = kGcRoot
+ };
+
+ enum class BakerReadBarrierWidth : uint8_t {
+ kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled).
+ kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled).
+ kLast = kNarrow
+ };
+
+ static constexpr size_t kBitsForBakerReadBarrierKind =
+ MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
+ static constexpr size_t kBitsForRegister = 4u;
+ using BakerReadBarrierKindField =
+ BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
+ using BakerReadBarrierFirstRegField =
+ BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>;
+ using BakerReadBarrierSecondRegField =
+ BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>;
+ static constexpr size_t kBitsForBakerReadBarrierWidth =
+ MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast));
+ using BakerReadBarrierWidthField = BitField<BakerReadBarrierWidth,
+ kBitsForBakerReadBarrierKind + 2 * kBitsForRegister,
+ kBitsForBakerReadBarrierWidth>;
+
+ static void CheckValidReg(uint32_t reg) {
+ DCHECK(reg < 12u && reg != kBakerCcEntrypointRegister) << reg;
+ }
+
+ void CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data);
+
void SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value);
static uint32_t GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset);
diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc
index e7b11bd16b..2c22a352c2 100644
--- a/compiler/linker/arm/relative_patcher_thumb2_test.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc
@@ -16,15 +16,12 @@
#include "linker/arm/relative_patcher_thumb2.h"
-#include "arch/arm/instruction_set_features_arm.h"
#include "base/casts.h"
#include "linker/relative_patcher_test.h"
#include "lock_word.h"
#include "mirror/array-inl.h"
#include "mirror/object.h"
#include "oat_quick_method_header.h"
-#include "optimizing/code_generator_arm_vixl.h"
-#include "optimizing/optimizing_unit_test.h"
namespace art {
namespace linker {
@@ -192,42 +189,9 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest {
return result.second - 1 /* thumb mode */;
}
- std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
- /*out*/ std::string* debug_name = nullptr) {
- OptimizingUnitTestHelper helper;
- HGraph* graph = helper.CreateGraph();
- std::string error_msg;
- ArmFeaturesUniquePtr features =
- ArmInstructionSetFeatures::FromVariant("default", &error_msg);
- CompilerOptions options;
- arm::CodeGeneratorARMVIXL codegen(graph, *features, options);
- ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
- codegen.EmitThunkCode(patch, &code, debug_name);
- return std::vector<uint8_t>(code.begin(), code.end());
- }
-
- void AddCompiledMethod(
- MethodReference method_ref,
- const ArrayRef<const uint8_t>& code,
- const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
- RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
-
- // Make sure the ThunkProvider has all the necessary thunks.
- for (const LinkerPatch& patch : patches) {
- if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
- patch.GetType() == LinkerPatch::Type::kCallRelative) {
- std::string debug_name;
- std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
- thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
- }
- }
- }
-
std::vector<uint8_t> CompileMethodCallThunk() {
- LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
- /* target_dex_file*/ nullptr,
- /* target_method_idx */ 0u);
- return CompileThunk(patch);
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey();
+ return static_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
uint32_t MethodCallThunkSize() {
@@ -264,38 +228,27 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest {
void TestStringReference(uint32_t string_offset);
void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
- static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
- uint32_t holder_reg,
- bool narrow) {
- return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow);
- }
-
- static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
- return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg);
- }
-
- static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
- return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow);
- }
-
std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg,
uint32_t holder_reg,
bool narrow) {
const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
- return CompileThunk(patch);
+ 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
+ return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
- return CompileThunk(patch);
+ 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg));
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
+ return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow));
- return CompileThunk(patch);
+ 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow));
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
+ return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
uint32_t GetOutputInsn32(uint32_t offset) {
@@ -641,7 +594,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref
const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
base_reg, holder_reg, /* narrow */ false);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
@@ -743,7 +696,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t r
const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
base_reg, holder_reg, /* narrow */ true);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
@@ -856,7 +809,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) {
constexpr uint32_t kLiteralOffset1 = 6u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -924,7 +877,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) {
constexpr uint32_t kLiteralOffset1 = 4u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -954,7 +907,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast
constexpr uint32_t kLiteralOffset1 = 6u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -1040,7 +993,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerArray) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
+ kLiteralOffset, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1121,7 +1074,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
+ kLiteralOffset,
+ Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1180,7 +1134,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
+ kLiteralOffset,
+ Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1227,7 +1182,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) {
patches.reserve(num_patches);
const uint32_t ldr =
kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12);
- uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
+ uint32_t encoded_data =
+ Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
for (size_t i = 0; i != num_patches; ++i) {
PushBackInsn(&code, ldr);
PushBackInsn(&code, kBneWPlus0);
@@ -1308,8 +1264,10 @@ TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) {
ldr1, kBneWPlus0, // First GC root LDR with read barrier.
ldr2, kBneWPlus0, // Second GC root LDR with read barrier.
});
- uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
- uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
+ uint32_t encoded_data1 =
+ Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
+ uint32_t encoded_data2 =
+ Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
const LinkerPatch last_method_patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc
index 135e39d100..b268204b4a 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64.cc
@@ -82,10 +82,9 @@ inline uint32_t MaxExtraSpace(size_t num_adrp, size_t code_size) {
} // anonymous namespace
-Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider,
+Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider,
const Arm64InstructionSetFeatures* features)
- : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kArm64),
+ : ArmBaseRelativePatcher(provider, InstructionSet::kArm64),
fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()),
reserved_adrp_thunks_(0u),
processed_adrp_thunks_(0u) {
@@ -314,6 +313,44 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod
uint32_t insn = GetInsn(code, literal_offset);
DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched)
ThunkKey key = GetBakerThunkKey(patch);
+ if (kIsDebugBuild) {
+ const uint32_t encoded_data = key.GetCustomValue1();
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ // Check that the next instruction matches the expected LDR.
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ DCHECK_GE(code->size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn(code, literal_offset + 4u);
+ // LDR (immediate) with correct base_reg.
+ CheckValidReg(next_insn & 0x1fu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5));
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ DCHECK_GE(code->size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn(code, literal_offset + 4u);
+ // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL),
+ // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2].
+ CheckValidReg(next_insn & 0x1fu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5));
+ CheckValidReg((next_insn >> 16) & 0x1f); // Check index register
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ DCHECK_GE(literal_offset, 4u);
+ uint32_t prev_insn = GetInsn(code, literal_offset - 4u);
+ // LDR (immediate) with correct root_reg.
+ const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
+ UNREACHABLE();
+ }
+ }
uint32_t target_offset = GetThunkTargetOffset(key, patch_offset);
DCHECK_ALIGNED(target_offset, 4u);
uint32_t disp = target_offset - patch_offset;
@@ -322,6 +359,216 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod
SetInsn(code, literal_offset, insn);
}
+#define __ assembler.GetVIXLAssembler()->
+
+static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler,
+ vixl::aarch64::Register base_reg,
+ vixl::aarch64::MemOperand& lock_word,
+ vixl::aarch64::Label* slow_path) {
+ using namespace vixl::aarch64; // NOLINT(build/namespaces)
+ // Load the lock word containing the rb_state.
+ __ Ldr(ip0.W(), lock_word);
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path);
+ static_assert(
+ BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET,
+ "Field and array LDR offsets must be the same to reuse the same code.");
+ // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning).
+ static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
+ "Field LDR must be 1 instruction (4B) before the return address label; "
+ " 2 instructions (8B) for heap poisoning.");
+ __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ __ Add(base_reg, base_reg, Operand(ip0, LSR, 32));
+ __ Br(lr); // And return back to the function.
+ // Note: The fake dependency is unnecessary for the slow path.
+}
+
+// Load the read barrier introspection entrypoint in register `entrypoint`.
+static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler,
+ vixl::aarch64::Register entrypoint) {
+ using vixl::aarch64::MemOperand;
+ using vixl::aarch64::ip0;
+ // Thread Register.
+ const vixl::aarch64::Register tr = vixl::aarch64::x19;
+
+ // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection.
+ DCHECK_EQ(ip0.GetCode(), 16u);
+ const int32_t entry_point_offset =
+ Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode());
+ __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
+}
+
+void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler,
+ uint32_t encoded_data) {
+ using namespace vixl::aarch64; // NOLINT(build/namespaces)
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ // Check if the holder is gray and, if not, add fake dependency to the base register
+ // and return to the LDR instruction to load the reference. Otherwise, use introspection
+ // to load the reference and call the entrypoint (in IP1) that performs further checks
+ // on the reference and marks it if needed.
+ auto base_reg =
+ Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ auto holder_reg =
+ Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data));
+ CheckValidReg(holder_reg.GetCode());
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip0, ip1);
+ // If base_reg differs from holder_reg, the offset was too large and we must have
+ // emitted an explicit null check before the load. Otherwise, we need to null-check
+ // the holder as we do not necessarily do that check before going to the thunk.
+ vixl::aarch64::Label throw_npe;
+ if (holder_reg.Is(base_reg)) {
+ __ Cbz(holder_reg.W(), &throw_npe);
+ }
+ vixl::aarch64::Label slow_path;
+ MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
+ __ Bind(&slow_path);
+ MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
+ __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset.
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
+ __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset.
+ __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference.
+ // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
+ __ Br(ip1); // Jump to the entrypoint.
+ if (holder_reg.Is(base_reg)) {
+ // Add null check slow path. The stack map is at the address pointed to by LR.
+ __ Bind(&throw_npe);
+ int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value();
+ __ Ldr(ip0, MemOperand(/* Thread* */ vixl::aarch64::x19, offset));
+ __ Br(ip0);
+ }
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ auto base_reg =
+ Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip0, ip1);
+ vixl::aarch64::Label slow_path;
+ int32_t data_offset =
+ mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
+ MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
+ DCHECK_LT(lock_word.GetOffset(), 0);
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
+ __ Bind(&slow_path);
+ MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET);
+ __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset.
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
+ __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set).
+ __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create
+ // a switch case target based on the index register.
+ __ Mov(ip0, base_reg); // Move the base register to ip0.
+ __ Br(ip1); // Jump to the entrypoint's array switch case.
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
+ // and it does not have a forwarding address), call the correct introspection entrypoint;
+ // otherwise return the reference (or the extracted forwarding address).
+ // There is no gray bit check for GC roots.
+ auto root_reg =
+ Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(root_reg.GetCode());
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip0, ip1);
+ vixl::aarch64::Label return_label, not_marked, forwarding_address;
+ __ Cbz(root_reg, &return_label);
+ MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value());
+ __ Ldr(ip0.W(), lock_word);
+ __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, &not_marked);
+ __ Bind(&return_label);
+ __ Br(lr);
+ __ Bind(&not_marked);
+ __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1));
+ __ B(&forwarding_address, mi);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
+ // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to
+ // art_quick_read_barrier_mark_introspection_gc_roots.
+ __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET));
+ __ Mov(ip0.W(), root_reg);
+ __ Br(ip1);
+ __ Bind(&forwarding_address);
+ __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift);
+ __ Br(lr);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
+ UNREACHABLE();
+ }
+}
+
+std::vector<uint8_t> Arm64RelativePatcher::CompileThunk(const ThunkKey& key) {
+ ArenaPool pool;
+ ArenaAllocator allocator(&pool);
+ arm64::Arm64Assembler assembler(&allocator);
+
+ switch (key.GetType()) {
+ case ThunkType::kMethodCall: {
+ // The thunk just uses the entry point in the ArtMethod. This works even for calls
+ // to the generic JNI and interpreter trampolines.
+ Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset(
+ kArm64PointerSize).Int32Value());
+ assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0));
+ break;
+ }
+ case ThunkType::kBakerReadBarrier: {
+ CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1());
+ break;
+ }
+ }
+
+ // Ensure we emit the literal pool.
+ assembler.FinalizeCode();
+ std::vector<uint8_t> thunk_code(assembler.CodeSize());
+ MemoryRegion code(thunk_code.data(), thunk_code.size());
+ assembler.FinalizeInstructions(code);
+ return thunk_code;
+}
+
+std::string Arm64RelativePatcher::GetThunkDebugName(const ThunkKey& key) {
+ switch (key.GetType()) {
+ case ThunkType::kMethodCall:
+ return "MethodCallThunk";
+
+ case ThunkType::kBakerReadBarrier: {
+ uint32_t encoded_data = key.GetCustomValue1();
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ std::ostringstream oss;
+ oss << "BakerReadBarrierThunk";
+ switch (kind) {
+ case BakerReadBarrierKind::kField:
+ oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
+ << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
+ break;
+ case BakerReadBarrierKind::kArray:
+ oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ break;
+ case BakerReadBarrierKind::kGcRoot:
+ oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
+ break;
+ }
+ return oss.str();
+ }
+ }
+}
+
+#undef __
+
uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) {
switch (key.GetType()) {
case ThunkType::kMethodCall:
diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h
index 9dc289da44..8ba59976e7 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.h
+++ b/compiler/linker/arm64/relative_patcher_arm64.h
@@ -18,6 +18,8 @@
#define ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_
#include "base/array_ref.h"
+#include "base/bit_field.h"
+#include "base/bit_utils.h"
#include "linker/arm/relative_patcher_arm_base.h"
namespace art {
@@ -30,8 +32,29 @@ namespace linker {
class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher {
public:
- Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider,
+ static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
+ CheckValidReg(base_reg);
+ CheckValidReg(holder_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(holder_reg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ CheckValidReg(base_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
+ CheckValidReg(root_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
+ BakerReadBarrierFirstRegField::Encode(root_reg) |
+ BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg);
+ }
+
+ Arm64RelativePatcher(RelativePatcherTargetProvider* provider,
const Arm64InstructionSetFeatures* features);
uint32_t ReserveSpace(uint32_t offset,
@@ -52,10 +75,37 @@ class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher {
uint32_t patch_offset) OVERRIDE;
protected:
+ std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE;
+ std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE;
uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE;
uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE;
private:
+ static constexpr uint32_t kInvalidEncodedReg = /* sp/zr is invalid */ 31u;
+
+ enum class BakerReadBarrierKind : uint8_t {
+ kField, // Field get or array get with constant offset (i.e. constant index).
+ kArray, // Array get with index in register.
+ kGcRoot, // GC root load.
+ kLast = kGcRoot
+ };
+
+ static constexpr size_t kBitsForBakerReadBarrierKind =
+ MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
+ static constexpr size_t kBitsForRegister = 5u;
+ using BakerReadBarrierKindField =
+ BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
+ using BakerReadBarrierFirstRegField =
+ BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>;
+ using BakerReadBarrierSecondRegField =
+ BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>;
+
+ static void CheckValidReg(uint32_t reg) {
+ DCHECK(reg < 30u && reg != 16u && reg != 17u) << reg;
+ }
+
+ void CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data);
+
static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp);
static bool NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, uint32_t literal_offset,
diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc
index 393733dd0c..05459a2a82 100644
--- a/compiler/linker/arm64/relative_patcher_arm64_test.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc
@@ -16,15 +16,12 @@
#include "linker/arm64/relative_patcher_arm64.h"
-#include "arch/arm64/instruction_set_features_arm64.h"
#include "base/casts.h"
#include "linker/relative_patcher_test.h"
#include "lock_word.h"
#include "mirror/array-inl.h"
#include "mirror/object.h"
#include "oat_quick_method_header.h"
-#include "optimizing/code_generator_arm64.h"
-#include "optimizing/optimizing_unit_test.h"
namespace art {
namespace linker {
@@ -171,42 +168,9 @@ class Arm64RelativePatcherTest : public RelativePatcherTest {
return result.second;
}
- std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
- /*out*/ std::string* debug_name = nullptr) {
- OptimizingUnitTestHelper helper;
- HGraph* graph = helper.CreateGraph();
- std::string error_msg;
- Arm64FeaturesUniquePtr features =
- Arm64InstructionSetFeatures::FromVariant("default", &error_msg);
- CompilerOptions options;
- arm64::CodeGeneratorARM64 codegen(graph, *features, options);
- ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
- codegen.EmitThunkCode(patch, &code, debug_name);
- return std::vector<uint8_t>(code.begin(), code.end());
- }
-
- void AddCompiledMethod(
- MethodReference method_ref,
- const ArrayRef<const uint8_t>& code,
- const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
- RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
-
- // Make sure the ThunkProvider has all the necessary thunks.
- for (const LinkerPatch& patch : patches) {
- if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
- patch.GetType() == LinkerPatch::Type::kCallRelative) {
- std::string debug_name;
- std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
- thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
- }
- }
- }
-
std::vector<uint8_t> CompileMethodCallThunk() {
- LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
- /* target_dex_file*/ nullptr,
- /* target_method_idx */ 0u);
- return CompileThunk(patch);
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey();
+ return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
uint32_t MethodCallThunkSize() {
@@ -511,34 +475,25 @@ class Arm64RelativePatcherTest : public RelativePatcherTest {
TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
}
- static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
- return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
- }
-
- static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
- return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierArrayData(base_reg);
- }
-
- static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
- return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierGcRootData(root_reg);
- }
-
std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) {
const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
- return CompileThunk(patch);
+ 0u, Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
+ return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
- return CompileThunk(patch);
+ 0u, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg));
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
+ return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg));
- return CompileThunk(patch);
+ 0u, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg));
+ ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
+ return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
}
uint32_t GetOutputInsn(uint32_t offset) {
@@ -964,7 +919,8 @@ void Arm64RelativePatcherTest::TestBakerField(uint32_t offset, uint32_t ref_reg)
const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
+ uint32_t encoded_data =
+ Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
};
@@ -1049,7 +1005,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddle) {
constexpr uint32_t kLiteralOffset1 = 4;
const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data =
+ Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1109,7 +1066,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkBeforeFiller) {
constexpr uint32_t kLiteralOffset1 = 0;
const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data =
+ Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1138,7 +1096,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddleUnreachableFr
constexpr uint32_t kLiteralOffset1 = 4;
const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data =
+ Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1211,7 +1170,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerArray) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
+ kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1288,7 +1247,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerGcRoot) {
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg)),
+ kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1384,8 +1343,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerAndMethodCallInteraction) {
kNopInsn, kNopInsn, // Padding before second GC root read barrier.
ldr2, kCbnzIP1Plus0Insn, // Second GC root LDR with read barrier.
});
- uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
- uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
+ uint32_t encoded_data1 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
+ uint32_t encoded_data2 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
const LinkerPatch last_method_patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
diff --git a/compiler/linker/linker_patch.h b/compiler/linker/linker_patch.h
index 7b35fd9b0c..710d8a690a 100644
--- a/compiler/linker/linker_patch.h
+++ b/compiler/linker/linker_patch.h
@@ -141,7 +141,7 @@ class LinkerPatch {
static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset,
uint32_t custom_value1 = 0u,
uint32_t custom_value2 = 0u) {
- LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, /* target_dex_file */ nullptr);
+ LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, nullptr);
patch.baker_custom_value1_ = custom_value1;
patch.baker_custom_value2_ = custom_value2;
return patch;
diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc
index b82d15230d..13877f8f12 100644
--- a/compiler/linker/relative_patcher.cc
+++ b/compiler/linker/relative_patcher.cc
@@ -43,8 +43,7 @@ namespace linker {
std::unique_ptr<RelativePatcher> RelativePatcher::Create(
InstructionSet instruction_set,
const InstructionSetFeatures* features,
- RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider) {
+ RelativePatcherTargetProvider* provider) {
class RelativePatcherNone FINAL : public RelativePatcher {
public:
RelativePatcherNone() { }
@@ -93,8 +92,7 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create(
};
UNUSED(features);
- UNUSED(thunk_provider);
- UNUSED(target_provider);
+ UNUSED(provider);
switch (instruction_set) {
#ifdef ART_ENABLE_CODEGEN_x86
case InstructionSet::kX86:
@@ -108,15 +106,12 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create(
case InstructionSet::kArm:
// Fall through: we generate Thumb2 code for "arm".
case InstructionSet::kThumb2:
- return std::unique_ptr<RelativePatcher>(
- new Thumb2RelativePatcher(thunk_provider, target_provider));
+ return std::unique_ptr<RelativePatcher>(new Thumb2RelativePatcher(provider));
#endif
#ifdef ART_ENABLE_CODEGEN_arm64
case InstructionSet::kArm64:
return std::unique_ptr<RelativePatcher>(
- new Arm64RelativePatcher(thunk_provider,
- target_provider,
- features->AsArm64InstructionSetFeatures()));
+ new Arm64RelativePatcher(provider, features->AsArm64InstructionSetFeatures()));
#endif
#ifdef ART_ENABLE_CODEGEN_mips
case InstructionSet::kMips:
diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h
index 06c7e70d23..b58e3dffbd 100644
--- a/compiler/linker/relative_patcher.h
+++ b/compiler/linker/relative_patcher.h
@@ -39,27 +39,6 @@ class LinkerPatch;
class OutputStream;
/**
- * @class RelativePatcherThunkProvider
- * @brief Interface for providing method offsets for relative call targets.
- */
-class RelativePatcherThunkProvider {
- public:
- /**
- * Get the code and debug name of a thunk needed by the given linker patch.
- *
- * @param patch The patch for which we need to retrieve the thunk code.
- * @param code A variable to receive the code of the thunk. This code must not be empty.
- * @param debug_name A variable to receive the debug name of the thunk.
- */
- virtual void GetThunkCode(const LinkerPatch& patch,
- /*out*/ ArrayRef<const uint8_t>* code,
- /*out*/ std::string* debug_name) = 0;
-
- protected:
- virtual ~RelativePatcherThunkProvider() { }
-};
-
-/**
* @class RelativePatcherTargetProvider
* @brief Interface for providing method offsets for relative call targets.
*/
@@ -91,10 +70,8 @@ class RelativePatcherTargetProvider {
class RelativePatcher {
public:
static std::unique_ptr<RelativePatcher> Create(
- InstructionSet instruction_set,
- const InstructionSetFeatures* features,
- RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider);
+ InstructionSet instruction_set, const InstructionSetFeatures* features,
+ RelativePatcherTargetProvider* provider);
virtual ~RelativePatcher() { }
diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h
index af8dc4dbc9..d21f2795b9 100644
--- a/compiler/linker/relative_patcher_test.h
+++ b/compiler/linker/relative_patcher_test.h
@@ -58,10 +58,7 @@ class RelativePatcherTest : public testing::Test {
instruction_set_(instruction_set),
features_(InstructionSetFeatures::FromVariant(instruction_set, variant, &error_msg_)),
method_offset_map_(),
- patcher_(RelativePatcher::Create(instruction_set,
- features_.get(),
- &thunk_provider_,
- &method_offset_map_)),
+ patcher_(RelativePatcher::Create(instruction_set, features_.get(), &method_offset_map_)),
bss_begin_(0u),
compiled_method_refs_(),
compiled_methods_(),
@@ -251,72 +248,6 @@ class RelativePatcherTest : public testing::Test {
LOG(ERROR) << " " << diff_indicator_str;
}
- class ThunkProvider : public RelativePatcherThunkProvider {
- public:
- ThunkProvider() {}
-
- void SetThunkCode(const LinkerPatch& patch,
- ArrayRef<const uint8_t> code,
- const std::string& debug_name) {
- thunk_map_.emplace(ThunkKey(patch), ThunkValue(code, debug_name));
- }
-
- void GetThunkCode(const LinkerPatch& patch,
- /*out*/ ArrayRef<const uint8_t>* code,
- /*out*/ std::string* debug_name) OVERRIDE {
- auto it = thunk_map_.find(ThunkKey(patch));
- CHECK(it != thunk_map_.end());
- const ThunkValue& value = it->second;
- CHECK(code != nullptr);
- *code = value.GetCode();
- CHECK(debug_name != nullptr);
- *debug_name = value.GetDebugName();
- }
-
- private:
- class ThunkKey {
- public:
- explicit ThunkKey(const LinkerPatch& patch)
- : type_(patch.GetType()),
- custom_value1_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch
- ? patch.GetBakerCustomValue1() : 0u),
- custom_value2_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch
- ? patch.GetBakerCustomValue2() : 0u) {
- CHECK(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
- patch.GetType() == LinkerPatch::Type::kCallRelative);
- }
-
- bool operator<(const ThunkKey& other) const {
- if (custom_value1_ != other.custom_value1_) {
- return custom_value1_ < other.custom_value1_;
- }
- if (custom_value2_ != other.custom_value2_) {
- return custom_value2_ < other.custom_value2_;
- }
- return type_ < other.type_;
- }
-
- private:
- const LinkerPatch::Type type_;
- const uint32_t custom_value1_;
- const uint32_t custom_value2_;
- };
-
- class ThunkValue {
- public:
- ThunkValue(ArrayRef<const uint8_t> code, const std::string& debug_name)
- : code_(code.begin(), code.end()), debug_name_(debug_name) {}
- ArrayRef<const uint8_t> GetCode() const { return ArrayRef<const uint8_t>(code_); }
- const std::string& GetDebugName() const { return debug_name_; }
-
- private:
- const std::vector<uint8_t> code_;
- const std::string debug_name_;
- };
-
- std::map<ThunkKey, ThunkValue> thunk_map_;
- };
-
// Map method reference to assinged offset.
// Wrap the map in a class implementing RelativePatcherTargetProvider.
class MethodOffsetMap FINAL : public RelativePatcherTargetProvider {
@@ -341,7 +272,6 @@ class RelativePatcherTest : public testing::Test {
std::string error_msg_;
InstructionSet instruction_set_;
std::unique_ptr<const InstructionSetFeatures> features_;
- ThunkProvider thunk_provider_;
MethodOffsetMap method_offset_map_;
std::unique_ptr<RelativePatcher> patcher_;
uint32_t bss_begin_;