Change interface conflict stub to take the interface method.
To avoid doing dex cache lookup, pass the interface method instead. This
costs a few hundred KBs on speed compiled APKs (< 0.5% code size), but
improves performance when hitting a conflict (as seen on dogfood data).
For nterp, we currently pass the conflict method instead of the
interface method. We need to handle default methods before optimizing
it.
This removes our last use of dex cache in compiled code. A follow-up CL
will remove the NeedsDexCacheOfDeclaringClass from HInvokeInterface.
Test: test.py
Change-Id: I3cdd4543ad7d904b3e81950af46a48a48af6991a
diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc
index 9ff1f73..c2ae0e0 100644
--- a/compiler/optimizing/code_generator.cc
+++ b/compiler/optimizing/code_generator.cc
@@ -923,8 +923,7 @@
return GetBootImageOffsetImpl(string.Ptr(), ImageHeader::kSectionObjects);
}
-uint32_t CodeGenerator::GetBootImageOffset(HInvokeStaticOrDirect* invoke) {
- DCHECK_EQ(invoke->GetMethodLoadKind(), MethodLoadKind::kBootImageRelRo);
+uint32_t CodeGenerator::GetBootImageOffset(HInvoke* invoke) {
ArtMethod* method = invoke->GetResolvedMethod();
DCHECK(method != nullptr);
return GetBootImageOffsetImpl(method, ImageHeader::kSectionArtMethods);
diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h
index 1a01be9..22804a9 100644
--- a/compiler/optimizing/code_generator.h
+++ b/compiler/optimizing/code_generator.h
@@ -632,7 +632,7 @@
uint32_t GetBootImageOffset(HLoadClass* load_class);
uint32_t GetBootImageOffset(HLoadString* load_string);
- uint32_t GetBootImageOffset(HInvokeStaticOrDirect* invoke);
+ uint32_t GetBootImageOffset(HInvoke* invoke);
static void CreateSystemArrayCopyLocationSummary(HInvoke* invoke);
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 36040ca..007aa43 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -4351,6 +4351,10 @@
void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
HandleInvoke(invoke);
+ if (invoke->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
+ // We cannot request ip1 as it's blocked by the register allocator.
+ invoke->GetLocations()->SetInAt(invoke->GetNumberOfArguments() - 1, Location::Any());
+ }
}
void CodeGeneratorARM64::MaybeGenerateInlineCacheCheck(HInstruction* instruction,
@@ -4421,7 +4425,17 @@
MacroAssembler* masm = GetVIXLAssembler();
UseScratchRegisterScope scratch_scope(masm);
scratch_scope.Exclude(ip1);
- __ Mov(ip1, invoke->GetMethodReference().index);
+ if (invoke->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
+ Location interface_method = locations->InAt(invoke->GetNumberOfArguments() - 1);
+ if (interface_method.IsStackSlot()) {
+ __ Ldr(ip1, StackOperandFrom(receiver));
+ } else {
+ __ Mov(ip1, XRegisterFrom(interface_method));
+ }
+ } else {
+ codegen_->LoadMethod(
+ invoke->GetHiddenArgumentLoadKind(), Location::RegisterLocation(ip1.GetCode()), invoke);
+ }
__ Ldr(temp,
MemOperand(temp, mirror::Class::ImtPtrOffset(kArm64PointerSize).Uint32Value()));
@@ -4489,21 +4503,8 @@
return desired_dispatch_info;
}
-void CodeGeneratorARM64::GenerateStaticOrDirectCall(
- HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
- // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention.
- Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
- switch (invoke->GetMethodLoadKind()) {
- case MethodLoadKind::kStringInit: {
- uint32_t offset =
- GetThreadOffset<kArm64PointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
- // temp = thread->string_init_entrypoint
- __ Ldr(XRegisterFrom(temp), MemOperand(tr, offset));
- break;
- }
- case MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
- break;
+void CodeGeneratorARM64::LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke) {
+ switch (load_kind) {
case MethodLoadKind::kBootImageLinkTimePcRelative: {
DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
// Add ADRP with its PC-relative method patch.
@@ -4538,14 +4539,47 @@
EmitLdrOffsetPlaceholder(ldr_label, XRegisterFrom(temp), XRegisterFrom(temp));
break;
}
- case MethodLoadKind::kJitDirectAddress:
+ case MethodLoadKind::kJitDirectAddress: {
// Load method address from literal pool.
- __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress()));
+ __ Ldr(XRegisterFrom(temp),
+ DeduplicateUint64Literal(reinterpret_cast<uint64_t>(invoke->GetResolvedMethod())));
break;
+ }
+ case MethodLoadKind::kRuntimeCall: {
+ // Test situation, don't do anything.
+ break;
+ }
+ default: {
+ LOG(FATAL) << "Load kind should have already been handled " << load_kind;
+ UNREACHABLE();
+ }
+ }
+}
+
+void CodeGeneratorARM64::GenerateStaticOrDirectCall(
+ HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
+ // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention.
+ Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
+ switch (invoke->GetMethodLoadKind()) {
+ case MethodLoadKind::kStringInit: {
+ uint32_t offset =
+ GetThreadOffset<kArm64PointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
+ // temp = thread->string_init_entrypoint
+ __ Ldr(XRegisterFrom(temp), MemOperand(tr, offset));
+ break;
+ }
+ case MethodLoadKind::kRecursive: {
+ callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
+ break;
+ }
case MethodLoadKind::kRuntimeCall: {
GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path);
return; // No code pointer retrieval; the runtime performs the call directly.
}
+ default: {
+ LoadMethod(invoke->GetMethodLoadKind(), temp, invoke);
+ break;
+ }
}
auto call_code_pointer_member = [&](MemberOffset offset) {
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index 447c0b5..7ae46d7 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -689,6 +689,7 @@
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
ArtMethod* method) override;
+ void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
void GenerateStaticOrDirectCall(
HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
void GenerateVirtualCall(
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index 0940f42..85337ed 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -3432,7 +3432,10 @@
void LocationsBuilderARMVIXL::VisitInvokeInterface(HInvokeInterface* invoke) {
HandleInvoke(invoke);
// Add the hidden argument.
- invoke->GetLocations()->AddTemp(LocationFrom(r12));
+ if (invoke->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
+ // We cannot request r12 as it's blocked by the register allocator.
+ invoke->GetLocations()->SetInAt(invoke->GetNumberOfArguments() - 1, Location::Any());
+ }
}
void CodeGeneratorARMVIXL::MaybeGenerateInlineCacheCheck(HInstruction* instruction,
@@ -3468,7 +3471,6 @@
// TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
LocationSummary* locations = invoke->GetLocations();
vixl32::Register temp = RegisterFrom(locations->GetTemp(0));
- vixl32::Register hidden_reg = RegisterFrom(locations->GetTemp(1));
Location receiver = locations->InAt(0);
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
@@ -3509,11 +3511,10 @@
// LR = temp->GetEntryPoint();
GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, entry_point);
- // Set the hidden (in r12) argument. It is done here, right before a BLX to prevent other
- // instruction from clobbering it as they might use r12 as a scratch register.
- DCHECK(hidden_reg.Is(r12));
-
{
+ // Set the hidden (in r12) argument. It is done here, right before a BLX to prevent other
+ // instruction from clobbering it as they might use r12 as a scratch register.
+ Location hidden_reg = Location::RegisterLocation(r12.GetCode());
// The VIXL macro assembler may clobber any of the scratch registers that are available to it,
// so it checks if the application is using them (by passing them to the macro assembler
// methods). The following application of UseScratchRegisterScope corrects VIXL's notion of
@@ -3523,8 +3524,18 @@
// (to materialize the constant), since the destination register becomes available for such use
// internally for the duration of the macro instruction.
UseScratchRegisterScope temps(GetVIXLAssembler());
- temps.Exclude(hidden_reg);
- __ Mov(hidden_reg, invoke->GetMethodReference().index);
+ temps.Exclude(RegisterFrom(hidden_reg));
+ if (invoke->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
+ Location current_method = locations->InAt(invoke->GetNumberOfArguments() - 1);
+ if (current_method.IsStackSlot()) {
+ GetAssembler()->LoadFromOffset(
+ kLoadWord, RegisterFrom(hidden_reg), sp, current_method.GetStackIndex());
+ } else {
+ __ Mov(RegisterFrom(hidden_reg), RegisterFrom(current_method));
+ }
+ } else {
+ codegen_->LoadMethod(invoke->GetHiddenArgumentLoadKind(), hidden_reg, invoke);
+ }
}
{
// Ensure the pc position is recorded immediately after the `blx` instruction.
@@ -9069,20 +9080,9 @@
return desired_dispatch_info;
}
-void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall(
- HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
- Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
- switch (invoke->GetMethodLoadKind()) {
- case MethodLoadKind::kStringInit: {
- uint32_t offset =
- GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
- // temp = thread->string_init_entrypoint
- GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, offset);
- break;
- }
- case MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
- break;
+
+void CodeGeneratorARMVIXL::LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke) {
+ switch (load_kind) {
case MethodLoadKind::kBootImageLinkTimePcRelative: {
DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
PcRelativePatchInfo* labels = NewBootImageMethodPatch(invoke->GetResolvedMethodReference());
@@ -9106,13 +9106,44 @@
GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, temp_reg, /* offset*/ 0);
break;
}
- case MethodLoadKind::kJitDirectAddress:
- __ Mov(RegisterFrom(temp), Operand::From(invoke->GetMethodAddress()));
+ case MethodLoadKind::kJitDirectAddress: {
+ __ Mov(RegisterFrom(temp), Operand::From(invoke->GetResolvedMethod()));
break;
+ }
+ case MethodLoadKind::kRuntimeCall: {
+ // Test situation, don't do anything.
+ break;
+ }
+ default: {
+ LOG(FATAL) << "Load kind should have already been handled " << load_kind;
+ UNREACHABLE();
+ }
+ }
+}
+
+void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall(
+ HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
+ Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
+ switch (invoke->GetMethodLoadKind()) {
+ case MethodLoadKind::kStringInit: {
+ uint32_t offset =
+ GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
+ // temp = thread->string_init_entrypoint
+ GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, offset);
+ break;
+ }
+ case MethodLoadKind::kRecursive: {
+ callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
+ break;
+ }
case MethodLoadKind::kRuntimeCall: {
GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path);
return; // No code pointer retrieval; the runtime performs the call directly.
}
+ default: {
+ LoadMethod(invoke->GetMethodLoadKind(), temp, invoke);
+ break;
+ }
}
auto call_code_pointer_member = [&](MemberOffset offset) {
diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h
index 0453d20..12594ed 100644
--- a/compiler/optimizing/code_generator_arm_vixl.h
+++ b/compiler/optimizing/code_generator_arm_vixl.h
@@ -581,6 +581,7 @@
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
ArtMethod* method) override;
+ void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
void GenerateStaticOrDirectCall(
HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
void GenerateVirtualCall(
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index 86e6b95..d05c2d9 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -2564,6 +2564,16 @@
// Add one temporary for inline cache update.
invoke->GetLocations()->AddTemp(Location::RegisterLocation(EBP));
}
+
+ // For PC-relative load kinds the invoke has an extra input, the PC-relative address base.
+ if (IsPcRelativeMethodLoadKind(invoke->GetHiddenArgumentLoadKind())) {
+ invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister());
+ }
+
+ if (invoke->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
+ invoke->GetLocations()->SetInAt(invoke->GetNumberOfArguments() - 1,
+ Location::RequiresRegister());
+ }
}
void CodeGeneratorX86::MaybeGenerateInlineCacheCheck(HInstruction* instruction, Register klass) {
@@ -2608,8 +2618,12 @@
// Set the hidden argument. This is safe to do this here, as XMM7
// won't be modified thereafter, before the `call` instruction.
DCHECK_EQ(XMM7, hidden_reg);
- __ movl(temp, Immediate(invoke->GetMethodReference().index));
- __ movd(hidden_reg, temp);
+ if (invoke->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
+ __ movd(hidden_reg, locations->InAt(invoke->GetNumberOfArguments() - 1).AsRegister<Register>());
+ } else {
+ codegen_->LoadMethod(invoke->GetHiddenArgumentLoadKind(), locations->GetTemp(0), invoke);
+ __ movd(hidden_reg, temp);
+ }
if (receiver.IsStackSlot()) {
__ movl(temp, Address(ESP, receiver.GetStackIndex()));
@@ -5147,6 +5161,16 @@
return desired_dispatch_info;
}
+Register CodeGeneratorX86::GetInvokeExtraParameter(HInvoke* invoke, Register temp) {
+ if (invoke->IsInvokeStaticOrDirect()) {
+ return GetInvokeStaticOrDirectExtraParameter(invoke->AsInvokeStaticOrDirect(), temp);
+ }
+ DCHECK(invoke->IsInvokeInterface());
+ Location location =
+ invoke->GetLocations()->InAt(invoke->AsInvokeInterface()->GetSpecialInputIndex());
+ return location.AsRegister<Register>();
+}
+
Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
Register temp) {
Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
@@ -5172,6 +5196,50 @@
return location.AsRegister<Register>();
}
+void CodeGeneratorX86::LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke) {
+ switch (load_kind) {
+ case MethodLoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
+ Register base_reg = GetInvokeExtraParameter(invoke, temp.AsRegister<Register>());
+ __ leal(temp.AsRegister<Register>(),
+ Address(base_reg, CodeGeneratorX86::kPlaceholder32BitOffset));
+ RecordBootImageMethodPatch(invoke);
+ break;
+ }
+ case MethodLoadKind::kBootImageRelRo: {
+ size_t index = invoke->IsInvokeInterface()
+ ? invoke->AsInvokeInterface()->GetSpecialInputIndex()
+ : invoke->AsInvokeStaticOrDirect()->GetSpecialInputIndex();
+ Register base_reg = GetInvokeExtraParameter(invoke, temp.AsRegister<Register>());
+ __ movl(temp.AsRegister<Register>(), Address(base_reg, kPlaceholder32BitOffset));
+ RecordBootImageRelRoPatch(
+ invoke->InputAt(index)->AsX86ComputeBaseMethodAddress(),
+ GetBootImageOffset(invoke));
+ break;
+ }
+ case MethodLoadKind::kBssEntry: {
+ Register base_reg = GetInvokeExtraParameter(invoke, temp.AsRegister<Register>());
+ __ movl(temp.AsRegister<Register>(), Address(base_reg, kPlaceholder32BitOffset));
+ RecordMethodBssEntryPatch(invoke);
+ // No need for memory fence, thanks to the x86 memory model.
+ break;
+ }
+ case MethodLoadKind::kJitDirectAddress: {
+ __ movl(temp.AsRegister<Register>(),
+ Immediate(reinterpret_cast32<uint32_t>(invoke->GetResolvedMethod())));
+ break;
+ }
+ case MethodLoadKind::kRuntimeCall: {
+ // Test situation, don't do anything.
+ break;
+ }
+ default: {
+ LOG(FATAL) << "Load kind should have already been handled " << load_kind;
+ UNREACHABLE();
+ }
+ }
+}
+
void CodeGeneratorX86::GenerateStaticOrDirectCall(
HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
@@ -5183,42 +5251,17 @@
__ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(offset));
break;
}
- case MethodLoadKind::kRecursive:
+ case MethodLoadKind::kRecursive: {
callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
break;
- case MethodLoadKind::kBootImageLinkTimePcRelative: {
- DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
- Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
- temp.AsRegister<Register>());
- __ leal(temp.AsRegister<Register>(),
- Address(base_reg, CodeGeneratorX86::kPlaceholder32BitOffset));
- RecordBootImageMethodPatch(invoke);
- break;
}
- case MethodLoadKind::kBootImageRelRo: {
- Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
- temp.AsRegister<Register>());
- __ movl(temp.AsRegister<Register>(), Address(base_reg, kPlaceholder32BitOffset));
- RecordBootImageRelRoPatch(
- invoke->InputAt(invoke->GetSpecialInputIndex())->AsX86ComputeBaseMethodAddress(),
- GetBootImageOffset(invoke));
- break;
- }
- case MethodLoadKind::kBssEntry: {
- Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
- temp.AsRegister<Register>());
- __ movl(temp.AsRegister<Register>(), Address(base_reg, kPlaceholder32BitOffset));
- RecordMethodBssEntryPatch(invoke);
- // No need for memory fence, thanks to the x86 memory model.
- break;
- }
- case MethodLoadKind::kJitDirectAddress:
- __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
- break;
case MethodLoadKind::kRuntimeCall: {
GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path);
return; // No code pointer retrieval; the runtime performs the call directly.
}
+ default: {
+ LoadMethod(invoke->GetMethodLoadKind(), callee_method, invoke);
+ }
}
switch (invoke->GetCodePtrLocation()) {
@@ -5336,9 +5379,12 @@
__ Bind(&boot_image_other_patches_.back().label);
}
-void CodeGeneratorX86::RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke) {
+void CodeGeneratorX86::RecordBootImageMethodPatch(HInvoke* invoke) {
+ size_t index = invoke->IsInvokeInterface()
+ ? invoke->AsInvokeInterface()->GetSpecialInputIndex()
+ : invoke->AsInvokeStaticOrDirect()->GetSpecialInputIndex();
HX86ComputeBaseMethodAddress* method_address =
- invoke->InputAt(invoke->GetSpecialInputIndex())->AsX86ComputeBaseMethodAddress();
+ invoke->InputAt(index)->AsX86ComputeBaseMethodAddress();
boot_image_method_patches_.emplace_back(
method_address,
invoke->GetResolvedMethodReference().dex_file,
@@ -5346,10 +5392,13 @@
__ Bind(&boot_image_method_patches_.back().label);
}
-void CodeGeneratorX86::RecordMethodBssEntryPatch(HInvokeStaticOrDirect* invoke) {
+void CodeGeneratorX86::RecordMethodBssEntryPatch(HInvoke* invoke) {
+ size_t index = invoke->IsInvokeInterface()
+ ? invoke->AsInvokeInterface()->GetSpecialInputIndex()
+ : invoke->AsInvokeStaticOrDirect()->GetSpecialInputIndex();
DCHECK(IsSameDexFile(GetGraph()->GetDexFile(), *invoke->GetMethodReference().dex_file));
HX86ComputeBaseMethodAddress* method_address =
- invoke->InputAt(invoke->GetSpecialInputIndex())->AsX86ComputeBaseMethodAddress();
+ invoke->InputAt(index)->AsX86ComputeBaseMethodAddress();
// Add the patch entry and bind its label at the end of the instruction.
method_bss_entry_patches_.emplace_back(
method_address,
diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h
index b0575ba..0368de5 100644
--- a/compiler/optimizing/code_generator_x86.h
+++ b/compiler/optimizing/code_generator_x86.h
@@ -472,6 +472,7 @@
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
ArtMethod* method) override;
+ void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
// Generate a call to a static or direct method.
void GenerateStaticOrDirectCall(
HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
@@ -483,8 +484,8 @@
uint32_t intrinsic_data);
void RecordBootImageRelRoPatch(HX86ComputeBaseMethodAddress* method_address,
uint32_t boot_image_offset);
- void RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke);
- void RecordMethodBssEntryPatch(HInvokeStaticOrDirect* invoke);
+ void RecordBootImageMethodPatch(HInvoke* invoke);
+ void RecordMethodBssEntryPatch(HInvoke* invoke);
void RecordBootImageTypePatch(HLoadClass* load_class);
Label* NewTypeBssEntryPatch(HLoadClass* load_class);
void RecordBootImageStringPatch(HLoadString* load_string);
@@ -697,6 +698,7 @@
void EmitPcRelativeLinkerPatches(const ArenaDeque<X86PcRelativePatchInfo>& infos,
ArenaVector<linker::LinkerPatch>* linker_patches);
+ Register GetInvokeExtraParameter(HInvoke* invoke, Register temp);
Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp);
// Labels for each block that will be compiled.
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index 202b58b..3a39ee8 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -1001,22 +1001,8 @@
return desired_dispatch_info;
}
-void CodeGeneratorX86_64::GenerateStaticOrDirectCall(
- HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
- // All registers are assumed to be correctly set up.
-
- Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
- switch (invoke->GetMethodLoadKind()) {
- case MethodLoadKind::kStringInit: {
- // temp = thread->string_init_entrypoint
- uint32_t offset =
- GetThreadOffset<kX86_64PointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
- __ gs()->movq(temp.AsRegister<CpuRegister>(), Address::Absolute(offset, /* no_rip= */ true));
- break;
- }
- case MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
- break;
+void CodeGeneratorX86_64::LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke) {
+ switch (load_kind) {
case MethodLoadKind::kBootImageLinkTimePcRelative:
DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
__ leal(temp.AsRegister<CpuRegister>(),
@@ -1037,13 +1023,47 @@
// No need for memory fence, thanks to the x86-64 memory model.
break;
}
- case MethodLoadKind::kJitDirectAddress:
- Load64BitValue(temp.AsRegister<CpuRegister>(), invoke->GetMethodAddress());
+ case MethodLoadKind::kJitDirectAddress: {
+ Load64BitValue(temp.AsRegister<CpuRegister>(),
+ reinterpret_cast<int64_t>(invoke->GetResolvedMethod()));
break;
+ }
+ case MethodLoadKind::kRuntimeCall: {
+ // Test situation, don't do anything.
+ break;
+ }
+ default: {
+ LOG(FATAL) << "Load kind should have already been handled " << load_kind;
+ UNREACHABLE();
+ }
+ }
+}
+
+void CodeGeneratorX86_64::GenerateStaticOrDirectCall(
+ HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
+ // All registers are assumed to be correctly set up.
+
+ Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
+ switch (invoke->GetMethodLoadKind()) {
+ case MethodLoadKind::kStringInit: {
+ // temp = thread->string_init_entrypoint
+ uint32_t offset =
+ GetThreadOffset<kX86_64PointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
+ __ gs()->movq(temp.AsRegister<CpuRegister>(), Address::Absolute(offset, /* no_rip= */ true));
+ break;
+ }
+ case MethodLoadKind::kRecursive: {
+ callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
+ break;
+ }
case MethodLoadKind::kRuntimeCall: {
GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path);
return; // No code pointer retrieval; the runtime performs the call directly.
}
+ default: {
+ LoadMethod(invoke->GetMethodLoadKind(), temp, invoke);
+ break;
+ }
}
switch (invoke->GetCodePtrLocation()) {
@@ -1147,13 +1167,13 @@
__ Bind(&boot_image_other_patches_.back().label);
}
-void CodeGeneratorX86_64::RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke) {
+void CodeGeneratorX86_64::RecordBootImageMethodPatch(HInvoke* invoke) {
boot_image_method_patches_.emplace_back(invoke->GetResolvedMethodReference().dex_file,
invoke->GetResolvedMethodReference().index);
__ Bind(&boot_image_method_patches_.back().label);
}
-void CodeGeneratorX86_64::RecordMethodBssEntryPatch(HInvokeStaticOrDirect* invoke) {
+void CodeGeneratorX86_64::RecordMethodBssEntryPatch(HInvoke* invoke) {
DCHECK(IsSameDexFile(GetGraph()->GetDexFile(), *invoke->GetMethodReference().dex_file));
method_bss_entry_patches_.emplace_back(invoke->GetMethodReference().dex_file,
invoke->GetMethodReference().index);
@@ -2711,6 +2731,10 @@
void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
HandleInvoke(invoke);
// Add the hidden argument.
+ if (invoke->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
+ invoke->GetLocations()->SetInAt(invoke->GetNumberOfArguments() - 1,
+ Location::RegisterLocation(RAX));
+ }
invoke->GetLocations()->AddTemp(Location::RegisterLocation(RAX));
}
@@ -2744,7 +2768,6 @@
// TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
LocationSummary* locations = invoke->GetLocations();
CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
- CpuRegister hidden_reg = locations->GetTemp(1).AsRegister<CpuRegister>();
Location receiver = locations->InAt(0);
size_t class_offset = mirror::Object::ClassOffset().SizeValue();
@@ -2768,11 +2791,14 @@
codegen_->MaybeGenerateInlineCacheCheck(invoke, temp);
- // Set the hidden argument. This is safe to do this here, as RAX
- // won't be modified thereafter, before the `call` instruction.
- // We also di it after MaybeGenerateInlineCache that may use RAX.
- DCHECK_EQ(RAX, hidden_reg.AsRegister());
- codegen_->Load64BitValue(hidden_reg, invoke->GetMethodReference().index);
+ if (invoke->GetHiddenArgumentLoadKind() != MethodLoadKind::kRecursive) {
+ Location hidden_reg = locations->GetTemp(1);
+ // Set the hidden argument. This is safe to do this here, as RAX
+ // won't be modified thereafter, before the `call` instruction.
+ // We also di it after MaybeGenerateInlineCache that may use RAX.
+ DCHECK_EQ(RAX, hidden_reg.AsRegister<Register>());
+ codegen_->LoadMethod(invoke->GetHiddenArgumentLoadKind(), hidden_reg, invoke);
+ }
// temp = temp->GetAddressOfIMT()
__ movq(temp,
diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h
index 81988b4..c69c80a 100644
--- a/compiler/optimizing/code_generator_x86_64.h
+++ b/compiler/optimizing/code_generator_x86_64.h
@@ -452,6 +452,7 @@
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
ArtMethod* method) override;
+ void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
void GenerateStaticOrDirectCall(
HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
void GenerateVirtualCall(
@@ -459,8 +460,8 @@
void RecordBootImageIntrinsicPatch(uint32_t intrinsic_data);
void RecordBootImageRelRoPatch(uint32_t boot_image_offset);
- void RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke);
- void RecordMethodBssEntryPatch(HInvokeStaticOrDirect* invoke);
+ void RecordBootImageMethodPatch(HInvoke* invoke);
+ void RecordMethodBssEntryPatch(HInvoke* invoke);
void RecordBootImageTypePatch(HLoadClass* load_class);
Label* NewTypeBssEntryPatch(HLoadClass* load_class);
void RecordBootImageStringPatch(HLoadString* load_string);
diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc
index f917500..0531d72 100644
--- a/compiler/optimizing/instruction_builder.cc
+++ b/compiler/optimizing/instruction_builder.cc
@@ -1104,9 +1104,9 @@
}
}
HInvokeStaticOrDirect::DispatchInfo dispatch_info =
- HSharpening::SharpenInvokeStaticOrDirect(resolved_method,
- has_method_id,
- code_generator_);
+ HSharpening::SharpenLoadMethod(resolved_method,
+ has_method_id,
+ code_generator_);
if (dispatch_info.code_ptr_location == CodePtrLocation::kCallCriticalNative) {
graph_->SetHasDirectCriticalNativeCall(true);
}
@@ -1138,6 +1138,13 @@
/*vtable_index=*/ imt_or_vtable_index);
} else {
DCHECK_EQ(invoke_type, kInterface);
+ if (kIsDebugBuild) {
+ ScopedObjectAccess soa(Thread::Current());
+ DCHECK(resolved_method->GetDeclaringClass()->IsInterface());
+ }
+ MethodLoadKind load_kind =
+ HSharpening::SharpenLoadMethod(resolved_method, /* has_method_id= */ true, code_generator_)
+ .method_load_kind;
invoke = new (allocator_) HInvokeInterface(allocator_,
number_of_arguments,
return_type,
@@ -1145,7 +1152,8 @@
method_reference,
resolved_method,
resolved_method_reference,
- /*imt_index=*/ imt_or_vtable_index);
+ /*imt_index=*/ imt_or_vtable_index,
+ load_kind);
}
return HandleInvoke(invoke, operands, shorty, /* is_unresolved= */ false);
}
@@ -1669,6 +1677,12 @@
invoke->SetRawInputAt(argument_index, graph_->GetCurrentMethod());
}
+ if (invoke->IsInvokeInterface() &&
+ (invoke->AsInvokeInterface()->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive)) {
+ invoke->SetRawInputAt(invoke->AsInvokeInterface()->GetNumberOfArguments() - 1,
+ graph_->GetCurrentMethod());
+ }
+
return true;
}
diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc
index d616912..d7d5b59 100644
--- a/compiler/optimizing/instruction_simplifier.cc
+++ b/compiler/optimizing/instruction_simplifier.cc
@@ -2311,7 +2311,7 @@
// is unlikely that it exists. The most usual situation for such typed
// arraycopy methods is a direct pointer to the boot image.
invoke->SetDispatchInfo(
- HSharpening::SharpenInvokeStaticOrDirect(method, /* has_method_id= */ true, codegen_));
+ HSharpening::SharpenLoadMethod(method, /* has_method_id= */ true, codegen_));
}
}
}
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index b674937..9200689 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -4417,6 +4417,12 @@
kCallArtMethod,
};
+static inline bool IsPcRelativeMethodLoadKind(MethodLoadKind load_kind) {
+ return load_kind == MethodLoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == MethodLoadKind::kBootImageRelRo ||
+ load_kind == MethodLoadKind::kBssEntry;
+}
+
class HInvoke : public HVariableInputSizeInstruction {
public:
bool NeedsEnvironment() const override;
@@ -4738,9 +4744,7 @@
bool IsStringInit() const { return GetMethodLoadKind() == MethodLoadKind::kStringInit; }
bool HasMethodAddress() const { return GetMethodLoadKind() == MethodLoadKind::kJitDirectAddress; }
bool HasPcRelativeMethodLoadKind() const {
- return GetMethodLoadKind() == MethodLoadKind::kBootImageLinkTimePcRelative ||
- GetMethodLoadKind() == MethodLoadKind::kBootImageRelRo ||
- GetMethodLoadKind() == MethodLoadKind::kBssEntry;
+ return IsPcRelativeMethodLoadKind(GetMethodLoadKind());
}
QuickEntrypointEnum GetStringInitEntryPoint() const {
@@ -4941,10 +4945,11 @@
MethodReference method_reference,
ArtMethod* resolved_method,
MethodReference resolved_method_reference,
- uint32_t imt_index)
+ uint32_t imt_index,
+ MethodLoadKind load_kind)
: HInvoke(kInvokeInterface,
allocator,
- number_of_arguments,
+ number_of_arguments + (NeedsCurrentMethod(load_kind) ? 1 : 0),
0u,
return_type,
dex_pc,
@@ -4952,7 +4957,12 @@
resolved_method,
resolved_method_reference,
kInterface),
- imt_index_(imt_index) {
+ imt_index_(imt_index),
+ hidden_argument_load_kind_(load_kind) {
+ }
+
+ static bool NeedsCurrentMethod(MethodLoadKind load_kind) {
+ return load_kind == MethodLoadKind::kRecursive;
}
bool IsClonable() const override { return true; }
@@ -4967,7 +4977,16 @@
return true;
}
+ size_t GetSpecialInputIndex() const {
+ return GetNumberOfArguments();
+ }
+
+ void AddSpecialInput(HInstruction* input) {
+ InsertInputAt(GetSpecialInputIndex(), input);
+ }
+
uint32_t GetImtIndex() const { return imt_index_; }
+ MethodLoadKind GetHiddenArgumentLoadKind() const { return hidden_argument_load_kind_; }
DECLARE_INSTRUCTION(InvokeInterface);
@@ -4977,6 +4996,9 @@
private:
// Cached value of the resolved method, to avoid needing the mutator lock.
const uint32_t imt_index_;
+
+ // How the hidden argument (the interface method) is being loaded.
+ const MethodLoadKind hidden_argument_load_kind_;
};
class HNeg final : public HUnaryOperation {
diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc
index 3ea1918..17f37f0 100644
--- a/compiler/optimizing/pc_relative_fixups_x86.cc
+++ b/compiler/optimizing/pc_relative_fixups_x86.cc
@@ -207,6 +207,15 @@
base_added = true;
}
+ HInvokeInterface* invoke_interface = invoke->AsInvokeInterface();
+ if (invoke_interface != nullptr &&
+ IsPcRelativeMethodLoadKind(invoke_interface->GetHiddenArgumentLoadKind())) {
+ HX86ComputeBaseMethodAddress* method_address = GetPCRelativeBasePointer(invoke);
+ // Add the extra parameter.
+ invoke_interface->AddSpecialInput(method_address);
+ base_added = true;
+ }
+
// Ensure that we can load FP arguments from the constant area.
HInputsRef inputs = invoke->GetInputs();
for (size_t i = 0; i < inputs.size(); i++) {
diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc
index f570c60..3ffb24b 100644
--- a/compiler/optimizing/sharpening.cc
+++ b/compiler/optimizing/sharpening.cc
@@ -57,7 +57,7 @@
return compiler_options.IsImageClass(dex_file.StringByTypeIdx(klass->GetDexTypeIndex()));
}
-HInvokeStaticOrDirect::DispatchInfo HSharpening::SharpenInvokeStaticOrDirect(
+HInvokeStaticOrDirect::DispatchInfo HSharpening::SharpenLoadMethod(
ArtMethod* callee, bool has_method_id, CodeGenerator* codegen) {
if (kIsDebugBuild) {
ScopedObjectAccess soa(Thread::Current()); // Required for GetDeclaringClass below.
diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h
index b48cd4b..f71d9b5 100644
--- a/compiler/optimizing/sharpening.h
+++ b/compiler/optimizing/sharpening.h
@@ -30,7 +30,7 @@
class HSharpening {
public:
// Used by the builder and InstructionSimplifier.
- static HInvokeStaticOrDirect::DispatchInfo SharpenInvokeStaticOrDirect(
+ static HInvokeStaticOrDirect::DispatchInfo SharpenLoadMethod(
ArtMethod* callee, bool has_method_id, CodeGenerator* codegen);
// Used by the builder and the inliner.