diff options
34 files changed, 499 insertions, 347 deletions
diff --git a/compiler/optimizing/bounds_check_elimination_test.cc b/compiler/optimizing/bounds_check_elimination_test.cc index 48090a3de4..e383ec664b 100644 --- a/compiler/optimizing/bounds_check_elimination_test.cc +++ b/compiler/optimizing/bounds_check_elimination_test.cc @@ -646,8 +646,13 @@ static HGraph* BuildSSAGraph3(ArenaAllocator* allocator, HBasicBlock* block = new (allocator) HBasicBlock(graph); graph->AddBlock(block); entry->AddSuccessor(block); - HInstruction* new_array = new (allocator) - HNewArray(constant_10, 0, Primitive::kPrimInt, graph->GetDexFile(), kQuickAllocArray); + HInstruction* new_array = new (allocator) HNewArray( + constant_10, + graph->GetCurrentMethod(), + 0, + Primitive::kPrimInt, + graph->GetDexFile(), + kQuickAllocArray); block->AddInstruction(new_array); block->AddInstruction(new (allocator) HGoto()); diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc index f98029da03..cbd042901d 100644 --- a/compiler/optimizing/builder.cc +++ b/compiler/optimizing/builder.cc @@ -763,6 +763,11 @@ bool HGraphBuilder::BuildInvoke(const Instruction& instruction, } DCHECK_EQ(argument_index, number_of_arguments); + if (invoke->IsInvokeStaticOrDirect()) { + invoke->SetArgumentAt(argument_index, graph_->GetCurrentMethod()); + argument_index++; + } + if (clinit_check_requirement == HInvokeStaticOrDirect::ClinitCheckRequirement::kExplicit) { // Add the class initialization check as last input of `invoke`. DCHECK(clinit_check != nullptr); @@ -1045,6 +1050,7 @@ void HGraphBuilder::BuildFilledNewArray(uint32_t dex_pc, ? kQuickAllocArrayWithAccessCheck : kQuickAllocArray; HInstruction* object = new (arena_) HNewArray(length, + graph_->GetCurrentMethod(), dex_pc, type_index, *dex_compilation_unit_->GetDexFile(), @@ -2003,7 +2009,11 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 : kQuickAllocObject; current_block_->AddInstruction(new (arena_) HNewInstance( - dex_pc, type_index, *dex_compilation_unit_->GetDexFile(), entrypoint)); + graph_->GetCurrentMethod(), + dex_pc, + type_index, + *dex_compilation_unit_->GetDexFile(), + entrypoint)); UpdateLocal(instruction.VRegA(), current_block_->GetLastInstruction()); } break; @@ -2015,8 +2025,12 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 QuickEntrypointEnum entrypoint = NeedsAccessCheck(type_index) ? kQuickAllocArrayWithAccessCheck : kQuickAllocArray; - current_block_->AddInstruction(new (arena_) HNewArray( - length, dex_pc, type_index, *dex_compilation_unit_->GetDexFile(), entrypoint)); + current_block_->AddInstruction(new (arena_) HNewArray(length, + graph_->GetCurrentMethod(), + dex_pc, + type_index, + *dex_compilation_unit_->GetDexFile(), + entrypoint)); UpdateLocal(instruction.VRegA_22c(), current_block_->GetLastInstruction()); break; } diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index 08c0351eab..049b3e3a40 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -292,7 +292,6 @@ void CodeGenerator::CreateCommonInvokeLocationSummary( HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) { ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena(); LocationSummary* locations = new (allocator) LocationSummary(invoke, LocationSummary::kCall); - locations->AddTemp(visitor->GetMethodLocation()); for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) { HInstruction* input = invoke->InputAt(i); @@ -300,6 +299,20 @@ void CodeGenerator::CreateCommonInvokeLocationSummary( } locations->SetOut(visitor->GetReturnLocation(invoke->GetType())); + + if (invoke->IsInvokeStaticOrDirect()) { + HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect(); + if (call->IsStringInit()) { + locations->AddTemp(visitor->GetMethodLocation()); + } else if (call->IsRecursive()) { + locations->SetInAt(call->GetCurrentMethodInputIndex(), visitor->GetMethodLocation()); + } else { + locations->AddTemp(visitor->GetMethodLocation()); + locations->SetInAt(call->GetCurrentMethodInputIndex(), Location::RequiresRegister()); + } + } else { + locations->AddTemp(visitor->GetMethodLocation()); + } } void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const { diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index f7731069fb..69f9ca4686 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -1254,17 +1254,16 @@ void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invok IntrinsicLocationsBuilderARM intrinsic(GetGraph()->GetArena(), codegen_->GetInstructionSetFeatures()); if (intrinsic.TryDispatch(invoke)) { + LocationSummary* locations = invoke->GetLocations(); + if (locations->CanCall()) { + locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::RequiresRegister()); + } return; } HandleInvoke(invoke); } -void CodeGeneratorARM::LoadCurrentMethod(Register reg) { - DCHECK(RequiresCurrentMethod()); - __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); -} - static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { if (invoke->GetLocations()->Intrinsified()) { IntrinsicCodeGeneratorARM intrinsic(codegen); @@ -1283,9 +1282,9 @@ void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirec return; } - Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>(); - - codegen_->GenerateStaticOrDirectCall(invoke, temp); + LocationSummary* locations = invoke->GetLocations(); + codegen_->GenerateStaticOrDirectCall( + invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } @@ -1316,12 +1315,8 @@ void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { Location receiver = locations->InAt(0); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); // temp = object->GetClass(); - if (receiver.IsStackSlot()) { - __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); - __ LoadFromOffset(kLoadWord, temp, temp, class_offset); - } else { - __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); - } + DCHECK(receiver.IsRegister()); + __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); codegen_->MaybeRecordImplicitNullCheck(invoke); // temp = temp->GetMethodAt(method_offset); uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( @@ -2710,13 +2705,12 @@ void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); locations->SetOut(Location::RegisterLocation(R0)); } void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { InvokeRuntimeCallingConvention calling_convention; - codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1)); __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex()); codegen_->InvokeRuntime(GetThreadOffset<kArmWordSize>(instruction->GetEntrypoint()).Int32Value(), instruction, @@ -2729,14 +2723,13 @@ void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); locations->SetOut(Location::RegisterLocation(R0)); locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); } void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { InvokeRuntimeCallingConvention calling_convention; - codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2)); __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex()); codegen_->InvokeRuntime(GetThreadOffset<kArmWordSize>(instruction->GetEntrypoint()).Int32Value(), instruction, @@ -4206,9 +4199,7 @@ void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instr } } -void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Register temp) { - DCHECK_EQ(temp, kArtMethodRegister); - +void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { // TODO: Implement all kinds of calls: // 1) boot -> boot // 2) app -> boot @@ -4217,32 +4208,32 @@ void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, // Currently we implement the app -> app logic, which looks up in the resolve cache. if (invoke->IsStringInit()) { + Register reg = temp.AsRegister<Register>(); // temp = thread->string_init_entrypoint - __ LoadFromOffset(kLoadWord, temp, TR, invoke->GetStringInitOffset()); + __ LoadFromOffset(kLoadWord, reg, TR, invoke->GetStringInitOffset()); // LR = temp[offset_of_quick_compiled_code] - __ LoadFromOffset(kLoadWord, LR, temp, + __ LoadFromOffset(kLoadWord, LR, reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset( kArmWordSize).Int32Value()); // LR() __ blx(LR); + } else if (invoke->IsRecursive()) { + __ bl(GetFrameEntryLabel()); } else { - // temp = method; - LoadCurrentMethod(temp); - if (!invoke->IsRecursive()) { - // temp = temp->dex_cache_resolved_methods_; - __ LoadFromOffset( - kLoadWord, temp, temp, ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()); - // temp = temp[index_in_cache] - __ LoadFromOffset( - kLoadWord, temp, temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())); - // LR = temp[offset_of_quick_compiled_code] - __ LoadFromOffset(kLoadWord, LR, temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArmWordSize).Int32Value()); - // LR() - __ blx(LR); - } else { - __ bl(GetFrameEntryLabel()); - } + Register current_method = + invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex()).AsRegister<Register>(); + Register reg = temp.AsRegister<Register>(); + // reg = current_method->dex_cache_resolved_methods_; + __ LoadFromOffset( + kLoadWord, reg, current_method, ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()); + // reg = reg[index_in_cache] + __ LoadFromOffset( + kLoadWord, reg, reg, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())); + // LR = reg[offset_of_quick_compiled_code] + __ LoadFromOffset(kLoadWord, LR, reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kArmWordSize).Int32Value()); + // LR() + __ blx(LR); } DCHECK(!IsLeafMethod()); diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index d84f2d3d20..af2481661a 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -271,9 +271,6 @@ class CodeGeneratorARM : public CodeGenerator { // Helper method to move a 64bits value between two locations. void Move64(Location destination, Location source); - // Load current method into `reg`. - void LoadCurrentMethod(Register reg); - // Generate code to invoke a runtime entry point. void InvokeRuntime( int32_t offset, HInstruction* instruction, uint32_t dex_pc, SlowPathCode* slow_path); @@ -301,7 +298,7 @@ class CodeGeneratorARM : public CodeGenerator { Label* GetFrameEntryLabel() { return &frame_entry_label_; } - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Register temp); + void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp); private: // Labels for each block that will be compiled. diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 2f607f70a3..5c8fea38cd 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -484,7 +484,7 @@ Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type } Location InvokeDexCallingConventionVisitorARM64::GetMethodLocation() const { - return LocationFrom(x0); + return LocationFrom(kArtMethodRegister); } CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, @@ -2227,6 +2227,10 @@ void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* inv IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena()); if (intrinsic.TryDispatch(invoke)) { + LocationSummary* locations = invoke->GetLocations(); + if (locations->CanCall()) { + locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::RequiresRegister()); + } return; } @@ -2242,9 +2246,8 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codege return false; } -void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Register temp) { +void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention. - DCHECK(temp.Is(kArtMethodRegister)); size_t index_in_cache = GetCachePointerOffset(invoke->GetDexMethodIndex()); // TODO: Implement all kinds of calls: @@ -2255,30 +2258,30 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok // Currently we implement the app -> app logic, which looks up in the resolve cache. if (invoke->IsStringInit()) { + Register reg = XRegisterFrom(temp); // temp = thread->string_init_entrypoint - __ Ldr(temp.X(), MemOperand(tr, invoke->GetStringInitOffset())); + __ Ldr(reg.X(), MemOperand(tr, invoke->GetStringInitOffset())); // LR = temp->entry_point_from_quick_compiled_code_; __ Ldr(lr, MemOperand( - temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value())); + reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value())); // lr() __ Blr(lr); + } else if (invoke->IsRecursive()) { + __ Bl(&frame_entry_label_); } else { - // temp = method; - LoadCurrentMethod(temp.X()); - if (!invoke->IsRecursive()) { - // temp = temp->dex_cache_resolved_methods_; - __ Ldr(temp.W(), MemOperand(temp.X(), - ArtMethod::DexCacheResolvedMethodsOffset().Int32Value())); - // temp = temp[index_in_cache]; - __ Ldr(temp.X(), MemOperand(temp, index_in_cache)); - // lr = temp->entry_point_from_quick_compiled_code_; - __ Ldr(lr, MemOperand(temp.X(), ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArm64WordSize).Int32Value())); - // lr(); - __ Blr(lr); - } else { - __ Bl(&frame_entry_label_); - } + Register current_method = + XRegisterFrom(invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex())); + Register reg = XRegisterFrom(temp); + // temp = current_method->dex_cache_resolved_methods_; + __ Ldr(reg.W(), MemOperand(current_method.X(), + ArtMethod::DexCacheResolvedMethodsOffset().Int32Value())); + // temp = temp[index_in_cache]; + __ Ldr(reg.X(), MemOperand(reg, index_in_cache)); + // lr = temp->entry_point_from_quick_compiled_code_; + __ Ldr(lr, MemOperand(reg.X(), ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kArm64WordSize).Int32Value())); + // lr(); + __ Blr(lr); } DCHECK(!IsLeafMethod()); @@ -2294,8 +2297,9 @@ void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDir } BlockPoolsScope block_pools(GetVIXLAssembler()); - Register temp = XRegisterFrom(invoke->GetLocations()->GetTemp(0)); - codegen_->GenerateStaticOrDirectCall(invoke, temp); + LocationSummary* locations = invoke->GetLocations(); + codegen_->GenerateStaticOrDirectCall( + invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } @@ -2314,14 +2318,8 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { BlockPoolsScope block_pools(GetVIXLAssembler()); - // temp = object->GetClass(); - if (receiver.IsStackSlot()) { - __ Ldr(temp.W(), MemOperand(sp, receiver.GetStackIndex())); - __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset)); - } else { - DCHECK(receiver.IsRegister()); - __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset)); - } + DCHECK(receiver.IsRegister()); + __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset)); codegen_->MaybeRecordImplicitNullCheck(invoke); // temp = temp->GetMethodAt(method_offset); __ Ldr(temp, MemOperand(temp, method_offset)); @@ -2523,9 +2521,9 @@ void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) { new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0))); - locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2))); locations->SetOut(LocationFrom(x0)); locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1))); + locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(2))); CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); } @@ -2535,9 +2533,6 @@ void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) { InvokeRuntimeCallingConvention calling_convention; Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt); DCHECK(type_index.Is(w0)); - Register current_method = RegisterFrom(locations->GetTemp(1), Primitive::kPrimLong); - DCHECK(current_method.Is(x2)); - codegen_->LoadCurrentMethod(current_method.X()); __ Mov(type_index, instruction->GetTypeIndex()); codegen_->InvokeRuntime( GetThreadOffset<kArm64WordSize>(instruction->GetEntrypoint()).Int32Value(), @@ -2552,7 +2547,7 @@ void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) { new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0))); - locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(1))); + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1))); locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>(); } @@ -2561,9 +2556,6 @@ void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) LocationSummary* locations = instruction->GetLocations(); Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt); DCHECK(type_index.Is(w0)); - Register current_method = RegisterFrom(locations->GetTemp(1), Primitive::kPrimNot); - DCHECK(current_method.Is(w1)); - codegen_->LoadCurrentMethod(current_method.X()); __ Mov(type_index, instruction->GetTypeIndex()); codegen_->InvokeRuntime( GetThreadOffset<kArm64WordSize>(instruction->GetEntrypoint()).Int32Value(), @@ -2674,7 +2666,7 @@ void InstructionCodeGeneratorARM64::VisitParameterValue( void LocationsBuilderARM64::VisitCurrentMethod(HCurrentMethod* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetOut(LocationFrom(x0)); + locations->SetOut(LocationFrom(kArtMethodRegister)); } void InstructionCodeGeneratorARM64::VisitCurrentMethod( diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index c62ba951cd..124fe88abb 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -150,8 +150,6 @@ class InstructionCodeGeneratorARM64 : public HGraphVisitor { FOR_EACH_CONCRETE_INSTRUCTION(DECLARE_VISIT_INSTRUCTION) #undef DECLARE_VISIT_INSTRUCTION - void LoadCurrentMethod(XRegister reg); - Arm64Assembler* GetAssembler() const { return assembler_; } vixl::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->vixl_masm_; } @@ -344,7 +342,7 @@ class CodeGeneratorARM64 : public CodeGenerator { return false; } - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, vixl::Register temp); + void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp); private: // Labels for each block that will be compiled. diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 8a7b52e549..51f3822901 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -527,11 +527,6 @@ void CodeGeneratorX86::Bind(HBasicBlock* block) { __ Bind(GetLabelOf(block)); } -void CodeGeneratorX86::LoadCurrentMethod(Register reg) { - DCHECK(RequiresCurrentMethod()); - __ movl(reg, Address(ESP, kCurrentMethodStackOffset)); -} - Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const { switch (load->GetType()) { case Primitive::kPrimLong: @@ -1231,10 +1226,25 @@ void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invok IntrinsicLocationsBuilderX86 intrinsic(codegen_); if (intrinsic.TryDispatch(invoke)) { + LocationSummary* locations = invoke->GetLocations(); + if (locations->CanCall()) { + locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::RequiresRegister()); + } return; } HandleInvoke(invoke); + + if (codegen_->IsBaseline()) { + // Baseline does not have enough registers if the current method also + // needs a register. We therefore do not require a register for it, and let + // the code generation of the invoke handle it. + LocationSummary* locations = invoke->GetLocations(); + Location location = locations->InAt(invoke->GetCurrentMethodInputIndex()); + if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) { + locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation()); + } + } } static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) { @@ -1255,8 +1265,9 @@ void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirec return; } + LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( - invoke, invoke->GetLocations()->GetTemp(0).AsRegister<Register>()); + invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } @@ -1276,13 +1287,8 @@ void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) { LocationSummary* locations = invoke->GetLocations(); Location receiver = locations->InAt(0); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - // temp = object->GetClass(); - if (receiver.IsStackSlot()) { - __ movl(temp, Address(ESP, receiver.GetStackIndex())); - __ movl(temp, Address(temp, class_offset)); - } else { - __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset)); - } + DCHECK(receiver.IsRegister()); + __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset)); codegen_->MaybeRecordImplicitNullCheck(invoke); // temp = temp->GetMethodAt(method_offset); __ movl(temp, Address(temp, method_offset)); @@ -2961,14 +2967,12 @@ void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) { locations->SetOut(Location::RegisterLocation(EAX)); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); } void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) { InvokeRuntimeCallingConvention calling_convention; - codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(1)); __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex())); - __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint()))); codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); @@ -2981,13 +2985,12 @@ void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) { locations->SetOut(Location::RegisterLocation(EAX)); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); } void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) { InvokeRuntimeCallingConvention calling_convention; - codegen_->LoadCurrentMethod(calling_convention.GetRegisterAt(2)); __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex())); __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(instruction->GetEntrypoint()))); @@ -3201,7 +3204,7 @@ void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) { void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, - Register temp) { + Location temp) { // TODO: Implement all kinds of calls: // 1) boot -> boot // 2) app -> boot @@ -3211,25 +3214,34 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, if (invoke->IsStringInit()) { // temp = thread->string_init_entrypoint - __ fs()->movl(temp, Address::Absolute(invoke->GetStringInitOffset())); + Register reg = temp.AsRegister<Register>(); + __ fs()->movl(reg, Address::Absolute(invoke->GetStringInitOffset())); // (temp + offset_of_quick_compiled_code)() __ call(Address( - temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); + reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); + } else if (invoke->IsRecursive()) { + __ call(GetFrameEntryLabel()); } else { - // temp = method; - LoadCurrentMethod(temp); - if (!invoke->IsRecursive()) { - // temp = temp->dex_cache_resolved_methods_; - __ movl(temp, Address(temp, ArtMethod::DexCacheResolvedMethodsOffset().Int32Value())); - // temp = temp[index_in_cache] - __ movl(temp, Address(temp, - CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex()))); - // (temp + offset_of_quick_compiled_code)() - __ call(Address(temp, - ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); + Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex()); + + Register method_reg; + Register reg = temp.AsRegister<Register>(); + if (current_method.IsRegister()) { + method_reg = current_method.AsRegister<Register>(); } else { - __ call(GetFrameEntryLabel()); + DCHECK(IsBaseline()); + DCHECK(!current_method.IsValid()); + method_reg = reg; + __ movl(reg, Address(ESP, kCurrentMethodStackOffset)); } + // temp = temp->dex_cache_resolved_methods_; + __ movl(reg, Address(method_reg, ArtMethod::DexCacheResolvedMethodsOffset().Int32Value())); + // temp = temp[index_in_cache] + __ movl(reg, Address(reg, + CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex()))); + // (temp + offset_of_quick_compiled_code)() + __ call(Address(reg, + ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); } DCHECK(!IsLeafMethod()); diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 61827a45ab..faf3cf9ffa 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -263,7 +263,7 @@ class CodeGeneratorX86 : public CodeGenerator { void Move64(Location destination, Location source); // Generate a call to a static or direct method. - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Register temp); + void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp); // Emit a write barrier. void MarkGCCard(Register temp, @@ -272,8 +272,6 @@ class CodeGeneratorX86 : public CodeGenerator { Register value, bool value_can_be_null); - void LoadCurrentMethod(Register reg); - Label* GetLabelOf(HBasicBlock* block) const { return CommonGetLabelOf<Label>(block_labels_.GetRawStorage(), block); } diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index a2a3cf523c..4e4cef855d 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -360,7 +360,7 @@ inline Condition X86_64Condition(IfCondition cond) { } void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, - CpuRegister temp) { + Location temp) { // All registers are assumed to be correctly set up. // TODO: Implement all kinds of calls: @@ -371,26 +371,28 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo // Currently we implement the app -> app logic, which looks up in the resolve cache. if (invoke->IsStringInit()) { + CpuRegister reg = temp.AsRegister<CpuRegister>(); // temp = thread->string_init_entrypoint - __ gs()->movl(temp, Address::Absolute(invoke->GetStringInitOffset())); + __ gs()->movl(reg, Address::Absolute(invoke->GetStringInitOffset())); // (temp + offset_of_quick_compiled_code)() - __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset( + __ call(Address(reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset( kX86_64WordSize).SizeValue())); + } else if (invoke->IsRecursive()) { + __ call(&frame_entry_label_); } else { - // temp = method; - LoadCurrentMethod(temp); - if (!invoke->IsRecursive()) { - // temp = temp->dex_cache_resolved_methods_; - __ movl(temp, Address(temp, ArtMethod::DexCacheResolvedMethodsOffset().SizeValue())); - // temp = temp[index_in_cache] - __ movq(temp, Address( - temp, CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex()))); - // (temp + offset_of_quick_compiled_code)() - __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kX86_64WordSize).SizeValue())); - } else { - __ call(&frame_entry_label_); - } + LocationSummary* locations = invoke->GetLocations(); + CpuRegister reg = temp.AsRegister<CpuRegister>(); + CpuRegister current_method = + locations->InAt(invoke->GetCurrentMethodInputIndex()).AsRegister<CpuRegister>(); + // temp = temp->dex_cache_resolved_methods_; + __ movl(reg, Address( + current_method, ArtMethod::DexCacheResolvedMethodsOffset().SizeValue())); + // temp = temp[index_in_cache] + __ movq(reg, Address( + reg, CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex()))); + // (temp + offset_of_quick_compiled_code)() + __ call(Address(reg, ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kX86_64WordSize).SizeValue())); } DCHECK(!IsLeafMethod()); @@ -585,11 +587,6 @@ void CodeGeneratorX86_64::Bind(HBasicBlock* block) { __ Bind(GetLabelOf(block)); } -void CodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) { - DCHECK(RequiresCurrentMethod()); - __ movq(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset)); -} - Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const { switch (load->GetType()) { case Primitive::kPrimLong: @@ -1334,6 +1331,10 @@ void LocationsBuilderX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* in IntrinsicLocationsBuilderX86_64 intrinsic(codegen_); if (intrinsic.TryDispatch(invoke)) { + LocationSummary* locations = invoke->GetLocations(); + if (locations->CanCall()) { + locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::RequiresRegister()); + } return; } @@ -1358,9 +1359,9 @@ void InstructionCodeGeneratorX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDi return; } + LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( - invoke, - invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>()); + invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } @@ -1390,12 +1391,8 @@ void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) Location receiver = locations->InAt(0); size_t class_offset = mirror::Object::ClassOffset().SizeValue(); // temp = object->GetClass(); - if (receiver.IsStackSlot()) { - __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex())); - __ movl(temp, Address(temp, class_offset)); - } else { - __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset)); - } + DCHECK(receiver.IsRegister()); + __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset)); codegen_->MaybeRecordImplicitNullCheck(invoke); // temp = temp->GetMethodAt(method_offset); __ movq(temp, Address(temp, method_offset)); @@ -3020,13 +3017,12 @@ void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) { new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); locations->SetOut(Location::RegisterLocation(RAX)); } void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) { InvokeRuntimeCallingConvention calling_convention; - codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1))); codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)), instruction->GetTypeIndex()); __ gs()->call( @@ -3041,14 +3037,13 @@ void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) { new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); InvokeRuntimeCallingConvention calling_convention; locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); locations->SetOut(Location::RegisterLocation(RAX)); locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); } void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) { InvokeRuntimeCallingConvention calling_convention; - codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(2))); codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)), instruction->GetTypeIndex()); diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index c19e686c10..e46994c79e 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -263,8 +263,6 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Helper method to move a value between two locations. void Move(Location destination, Location source); - void LoadCurrentMethod(CpuRegister reg); - Label* GetLabelOf(HBasicBlock* block) const { return CommonGetLabelOf<Label>(block_labels_.GetRawStorage(), block); } @@ -277,7 +275,7 @@ class CodeGeneratorX86_64 : public CodeGenerator { return false; } - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, CpuRegister temp); + void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp); const X86_64InstructionSetFeatures& GetInstructionSetFeatures() const { return isa_features_; diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc index 5436ec2dd9..749bedf99e 100644 --- a/compiler/optimizing/intrinsics_arm.cc +++ b/compiler/optimizing/intrinsics_arm.cc @@ -101,7 +101,8 @@ class IntrinsicSlowPathARM : public SlowPathCodeARM { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), kArtMethodRegister); + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), + Location::RegisterLocation(kArtMethodRegister)); RecordPcInfo(codegen, invoke_, invoke_->GetDexPc()); } else { UNIMPLEMENTED(FATAL) << "Non-direct intrinsic slow-path not yet implemented"; diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index d1dc5b3843..c108ad5daa 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -110,7 +110,8 @@ class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), kArtMethodRegister); + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), + LocationFrom(kArtMethodRegister)); RecordPcInfo(codegen, invoke_, invoke_->GetDexPc()); } else { UNIMPLEMENTED(FATAL) << "Non-direct intrinsic slow-path not yet implemented"; diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index 5bbbc72020..424ac7c855 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -138,7 +138,8 @@ class IntrinsicSlowPathX86 : public SlowPathCodeX86 { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), EAX); + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), + Location::RegisterLocation(EAX)); RecordPcInfo(codegen, invoke_, invoke_->GetDexPc()); } else { UNIMPLEMENTED(FATAL) << "Non-direct intrinsic slow-path not yet implemented"; @@ -732,7 +733,8 @@ static void InvokeOutOfLineIntrinsic(CodeGeneratorX86* codegen, HInvoke* invoke) MoveArguments(invoke, codegen); DCHECK(invoke->IsInvokeStaticOrDirect()); - codegen->GenerateStaticOrDirectCall(invoke->AsInvokeStaticOrDirect(), EAX); + codegen->GenerateStaticOrDirectCall(invoke->AsInvokeStaticOrDirect(), + Location::RegisterLocation(EAX)); codegen->RecordPcInfo(invoke, invoke->GetDexPc()); // Copy the result back to the expected output. diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index d6c90ff510..891531435e 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -129,7 +129,8 @@ class IntrinsicSlowPathX86_64 : public SlowPathCodeX86_64 { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), CpuRegister(RDI)); + codegen->GenerateStaticOrDirectCall( + invoke_->AsInvokeStaticOrDirect(), Location::RegisterLocation(RDI)); RecordPcInfo(codegen, invoke_, invoke_->GetDexPc()); } else { UNIMPLEMENTED(FATAL) << "Non-direct intrinsic slow-path not yet implemented"; @@ -609,7 +610,8 @@ static void InvokeOutOfLineIntrinsic(CodeGeneratorX86_64* codegen, HInvoke* invo MoveArguments(invoke, codegen); DCHECK(invoke->IsInvokeStaticOrDirect()); - codegen->GenerateStaticOrDirectCall(invoke->AsInvokeStaticOrDirect(), CpuRegister(RDI)); + codegen->GenerateStaticOrDirectCall( + invoke->AsInvokeStaticOrDirect(), Location::RegisterLocation(RDI)); codegen->RecordPcInfo(invoke, invoke->GetDexPc()); // Copy the result back to the expected output. diff --git a/compiler/optimizing/locations.h b/compiler/optimizing/locations.h index 09bbb33042..f41a782fe6 100644 --- a/compiler/optimizing/locations.h +++ b/compiler/optimizing/locations.h @@ -481,7 +481,6 @@ class LocationSummary : public ArenaObject<kArenaAllocMisc> { bool intrinsified = false); void SetInAt(uint32_t at, Location location) { - DCHECK(inputs_.Get(at).IsUnallocated() || inputs_.Get(at).IsInvalid()); inputs_.Put(at, location); } @@ -525,6 +524,8 @@ class LocationSummary : public ArenaObject<kArenaAllocMisc> { return temps_.Size(); } + bool HasTemps() const { return !temps_.IsEmpty(); } + Location Out() const { return output_; } bool CanCall() const { return call_kind_ != kNoCall; } diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index 47927340f4..f87775e195 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -2528,7 +2528,9 @@ class HInvokeStaticOrDirect : public HInvoke { ClinitCheckRequirement clinit_check_requirement) : HInvoke(arena, number_of_arguments, - clinit_check_requirement == ClinitCheckRequirement::kExplicit ? 1u : 0u, + // There is one extra argument for the HCurrentMethod node, and + // potentially one other if the clinit check is explicit. + clinit_check_requirement == ClinitCheckRequirement::kExplicit ? 2u : 1u, return_type, dex_pc, dex_method_index, @@ -2550,6 +2552,7 @@ class HInvokeStaticOrDirect : public HInvoke { bool NeedsDexCache() const OVERRIDE { return !IsRecursive(); } bool IsStringInit() const { return string_init_offset_ != 0; } int32_t GetStringInitOffset() const { return string_init_offset_; } + uint32_t GetCurrentMethodInputIndex() const { return GetNumberOfArguments(); } // Is this instruction a call to a static method? bool IsStatic() const { @@ -2665,9 +2668,10 @@ class HInvokeInterface : public HInvoke { DISALLOW_COPY_AND_ASSIGN(HInvokeInterface); }; -class HNewInstance : public HExpression<0> { +class HNewInstance : public HExpression<1> { public: - HNewInstance(uint32_t dex_pc, + HNewInstance(HCurrentMethod* current_method, + uint32_t dex_pc, uint16_t type_index, const DexFile& dex_file, QuickEntrypointEnum entrypoint) @@ -2675,7 +2679,9 @@ class HNewInstance : public HExpression<0> { dex_pc_(dex_pc), type_index_(type_index), dex_file_(dex_file), - entrypoint_(entrypoint) {} + entrypoint_(entrypoint) { + SetRawInputAt(0, current_method); + } uint32_t GetDexPc() const OVERRIDE { return dex_pc_; } uint16_t GetTypeIndex() const { return type_index_; } @@ -2718,9 +2724,10 @@ class HNeg : public HUnaryOperation { DISALLOW_COPY_AND_ASSIGN(HNeg); }; -class HNewArray : public HExpression<1> { +class HNewArray : public HExpression<2> { public: HNewArray(HInstruction* length, + HCurrentMethod* current_method, uint32_t dex_pc, uint16_t type_index, const DexFile& dex_file, @@ -2731,6 +2738,7 @@ class HNewArray : public HExpression<1> { dex_file_(dex_file), entrypoint_(entrypoint) { SetRawInputAt(0, length); + SetRawInputAt(1, current_method); } uint32_t GetDexPc() const OVERRIDE { return dex_pc_; } diff --git a/compiler/optimizing/register_allocator.cc b/compiler/optimizing/register_allocator.cc index a381315bac..e38e49cd19 100644 --- a/compiler/optimizing/register_allocator.cc +++ b/compiler/optimizing/register_allocator.cc @@ -714,13 +714,15 @@ bool RegisterAllocator::TryAllocateFreeReg(LiveInterval* current) { if (defined_by != nullptr && !current->IsSplit()) { LocationSummary* locations = defined_by->GetLocations(); if (!locations->OutputCanOverlapWithInputs() && locations->Out().IsUnallocated()) { - for (HInputIterator it(defined_by); !it.Done(); it.Advance()) { + for (size_t i = 0, e = defined_by->InputCount(); i < e; ++i) { // Take the last interval of the input. It is the location of that interval // that will be used at `defined_by`. - LiveInterval* interval = it.Current()->GetLiveInterval()->GetLastSibling(); + LiveInterval* interval = defined_by->InputAt(i)->GetLiveInterval()->GetLastSibling(); // Note that interval may have not been processed yet. // TODO: Handle non-split intervals last in the work list. - if (interval->HasRegister() && interval->SameRegisterKind(*current)) { + if (locations->InAt(i).IsValid() + && interval->HasRegister() + && interval->SameRegisterKind(*current)) { // The input must be live until the end of `defined_by`, to comply to // the linear scan algorithm. So we use `defined_by`'s end lifetime // position to check whether the input is dead or is inactive after diff --git a/compiler/optimizing/ssa_liveness_analysis.cc b/compiler/optimizing/ssa_liveness_analysis.cc index d5f977feec..701dbb019b 100644 --- a/compiler/optimizing/ssa_liveness_analysis.cc +++ b/compiler/optimizing/ssa_liveness_analysis.cc @@ -242,7 +242,7 @@ void SsaLivenessAnalysis::ComputeLiveRanges() { HInstruction* input = current->InputAt(i); // Some instructions 'inline' their inputs, that is they do not need // to be materialized. - if (input->HasSsaIndex()) { + if (input->HasSsaIndex() && current->GetLocations()->InAt(i).IsValid()) { live_in->SetBit(input->GetSsaIndex()); input->GetLiveInterval()->AddUse(current, /* environment */ nullptr, i); } diff --git a/compiler/optimizing/ssa_liveness_analysis.h b/compiler/optimizing/ssa_liveness_analysis.h index 4667825a62..220ee6a8d0 100644 --- a/compiler/optimizing/ssa_liveness_analysis.h +++ b/compiler/optimizing/ssa_liveness_analysis.h @@ -394,7 +394,7 @@ class LiveInterval : public ArenaObject<kArenaAllocMisc> { first_range_->start_ = from; } else { // Instruction without uses. - DCHECK(!defined_by_->HasNonEnvironmentUses()); + DCHECK(first_use_ == nullptr); DCHECK(from == defined_by_->GetLifetimePosition()); first_range_ = last_range_ = range_search_start_ = new (allocator_) LiveRange(from, from + 2, nullptr); diff --git a/runtime/art_method.cc b/runtime/art_method.cc index fe26438eac..50d9860d82 100644 --- a/runtime/art_method.cc +++ b/runtime/art_method.cc @@ -428,7 +428,8 @@ void ArtMethod::Invoke(Thread* self, uint32_t* args, uint32_t args_size, JValue* // exception was thrown to force the activations to be removed from the // stack. Continue execution in the interpreter. self->ClearException(); - ShadowFrame* shadow_frame = self->GetAndClearDeoptimizationShadowFrame(result); + ShadowFrame* shadow_frame = self->PopStackedShadowFrame(kDeoptimizationShadowFrame); + result->SetJ(self->PopDeoptimizationReturnValue().GetJ()); self->SetTopOfStack(nullptr); self->SetTopOfShadowStack(shadow_frame); interpreter::EnterInterpreterFromDeoptimize(self, shadow_frame, result); diff --git a/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc b/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc index 3d42ea09d9..f1b54459df 100644 --- a/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc @@ -36,6 +36,7 @@ extern "C" NO_RETURN void artDeoptimize(Thread* self) SHARED_LOCKS_REQUIRED(Lock self->Dump(LOG(INFO)); } + self->PushAndClearDeoptimizationReturnValue(); self->SetException(Thread::GetDeoptimizationException()); self->QuickDeliverException(); } diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc index cc83db1e59..4f76ebdd40 100644 --- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc @@ -676,7 +676,7 @@ extern "C" uint64_t artQuickToInterpreterBridge(ArtMethod* method, Thread* self, ArtMethod* caller = QuickArgumentVisitor::GetCallingMethod(sp); if (UNLIKELY(Dbg::IsForcedInterpreterNeededForUpcall(self, caller))) { self->SetException(Thread::GetDeoptimizationException()); - self->SetDeoptimizationReturnValue(result); + self->SetDeoptimizationReturnValue(result, shorty[0] == 'L'); } // No need to restore the args since the method has already been run by the interpreter. diff --git a/runtime/entrypoints_order_test.cc b/runtime/entrypoints_order_test.cc index 963dd0265f..0a5ebfa81b 100644 --- a/runtime/entrypoints_order_test.cc +++ b/runtime/entrypoints_order_test.cc @@ -72,6 +72,8 @@ class EntrypointsOrderTest : public CommonRuntimeTest { EXPECT_OFFSET_DIFFP(Thread, tls32_, throwing_OutOfMemoryError, no_thread_suspension, 4); EXPECT_OFFSET_DIFFP(Thread, tls32_, no_thread_suspension, thread_exit_check_count, 4); EXPECT_OFFSET_DIFFP(Thread, tls32_, thread_exit_check_count, handling_signal_, 4); + EXPECT_OFFSET_DIFFP(Thread, tls32_, handling_signal_, + deoptimization_return_value_is_reference, 4); // TODO: Better connection. Take alignment into account. EXPECT_OFFSET_DIFF_GT3(Thread, tls32_.thread_exit_check_count, tls64_.trace_clock_base, 4, @@ -103,11 +105,11 @@ class EntrypointsOrderTest : public CommonRuntimeTest { EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, long_jump_context, instrumentation_stack, sizeof(void*)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, instrumentation_stack, debug_invoke_req, sizeof(void*)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, debug_invoke_req, single_step_control, sizeof(void*)); - EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, single_step_control, deoptimization_shadow_frame, + EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, single_step_control, stacked_shadow_frame_record, sizeof(void*)); - EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, deoptimization_shadow_frame, - shadow_frame_under_construction, sizeof(void*)); - EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, shadow_frame_under_construction, name, sizeof(void*)); + EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, stacked_shadow_frame_record, + deoptimization_return_value_stack, sizeof(void*)); + EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, deoptimization_return_value_stack, name, sizeof(void*)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, name, pthread_self, sizeof(void*)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, pthread_self, last_no_thread_suspension_cause, sizeof(void*)); diff --git a/runtime/instrumentation.cc b/runtime/instrumentation.cc index 4ced23d488..d37ddcb88b 100644 --- a/runtime/instrumentation.cc +++ b/runtime/instrumentation.cc @@ -1019,7 +1019,7 @@ TwoWordReturn Instrumentation::PopInstrumentationStackFrame(Thread* self, uintpt PrettyMethod(method).c_str(), return_value.GetJ()) << *self; } - self->SetDeoptimizationReturnValue(return_value); + self->SetDeoptimizationReturnValue(return_value, return_shorty == 'L'); return GetTwoWordSuccessValue(*return_pc, reinterpret_cast<uintptr_t>(GetQuickDeoptimizationEntryPoint())); } else { diff --git a/runtime/interpreter/interpreter_common.cc b/runtime/interpreter/interpreter_common.cc index 1ed1a649b8..3f2e6db716 100644 --- a/runtime/interpreter/interpreter_common.cc +++ b/runtime/interpreter/interpreter_common.cc @@ -517,7 +517,8 @@ bool DoCall(ArtMethod* called_method, Thread* self, ShadowFrame& shadow_frame, // Slow path. // We might need to do class loading, which incurs a thread state change to kNative. So // register the shadow frame as under construction and allow suspension again. - self->SetShadowFrameUnderConstruction(new_shadow_frame); + ScopedStackedShadowFramePusher pusher( + self, new_shadow_frame, kShadowFrameUnderConstruction); self->EndAssertNoThreadSuspension(old_cause); // We need to do runtime check on reference assignment. We need to load the shorty @@ -590,8 +591,6 @@ bool DoCall(ArtMethod* called_method, Thread* self, ShadowFrame& shadow_frame, break; } } - // We're done with the construction. - self->ClearShadowFrameUnderConstruction(); } else { // Fast path: no extra checks. if (is_range) { diff --git a/runtime/jvalue.h b/runtime/jvalue.h index b39567b297..6a6d1986dc 100644 --- a/runtime/jvalue.h +++ b/runtime/jvalue.h @@ -61,6 +61,8 @@ union PACKED(4) JValue { uint8_t GetZ() const { return z; } void SetZ(uint8_t new_z) { z = new_z; } + mirror::Object** GetGCRoot() { return &l; } + private: uint8_t z; int8_t b; diff --git a/runtime/mem_map.cc b/runtime/mem_map.cc index 6b9b5d1015..189306df5b 100644 --- a/runtime/mem_map.cc +++ b/runtime/mem_map.cc @@ -240,6 +240,22 @@ static bool CheckMapRequest(uint8_t* expected_ptr, void* actual_ptr, size_t byte return false; } +#if USE_ART_LOW_4G_ALLOCATOR +static inline void* TryMemMapLow4GB(void* ptr, size_t page_aligned_byte_count, int prot, int flags, + int fd) { + void* actual = mmap(ptr, page_aligned_byte_count, prot, flags, fd, 0); + if (actual != MAP_FAILED) { + // Since we didn't use MAP_FIXED the kernel may have mapped it somewhere not in the low + // 4GB. If this is the case, unmap and retry. + if (reinterpret_cast<uintptr_t>(actual) + page_aligned_byte_count >= 4 * GB) { + munmap(actual, page_aligned_byte_count); + actual = MAP_FAILED; + } + } + return actual; +} +#endif + MemMap* MemMap::MapAnonymous(const char* name, uint8_t* expected_ptr, size_t byte_count, int prot, bool low_4gb, bool reuse, std::string* error_msg) { #ifndef __LP64__ @@ -339,6 +355,14 @@ MemMap* MemMap::MapAnonymous(const char* name, uint8_t* expected_ptr, size_t byt ++it; } + // Try to see if we get lucky with this address since none of the ART maps overlap. + actual = TryMemMapLow4GB(reinterpret_cast<void*>(ptr), page_aligned_byte_count, prot, flags, + fd.get()); + if (actual != MAP_FAILED) { + next_mem_pos_ = reinterpret_cast<uintptr_t>(actual) + page_aligned_byte_count; + break; + } + if (4U * GB - ptr < page_aligned_byte_count) { // Not enough memory until 4GB. if (first_run) { @@ -368,17 +392,10 @@ MemMap* MemMap::MapAnonymous(const char* name, uint8_t* expected_ptr, size_t byt next_mem_pos_ = tail_ptr; // update early, as we break out when we found and mapped a region if (safe == true) { - actual = mmap(reinterpret_cast<void*>(ptr), page_aligned_byte_count, prot, flags, fd.get(), - 0); + actual = TryMemMapLow4GB(reinterpret_cast<void*>(ptr), page_aligned_byte_count, prot, flags, + fd.get()); if (actual != MAP_FAILED) { - // Since we didn't use MAP_FIXED the kernel may have mapped it somewhere not in the low - // 4GB. If this is the case, unmap and retry. - if (reinterpret_cast<uintptr_t>(actual) + page_aligned_byte_count < 4 * GB) { break; - } else { - munmap(actual, page_aligned_byte_count); - actual = MAP_FAILED; - } } } else { // Skip over last page. diff --git a/runtime/quick_exception_handler.cc b/runtime/quick_exception_handler.cc index 8c9782aefe..a10c5c8200 100644 --- a/runtime/quick_exception_handler.cc +++ b/runtime/quick_exception_handler.cc @@ -163,8 +163,8 @@ class DeoptimizeStackVisitor FINAL : public StackVisitor { : StackVisitor(self, context, StackVisitor::StackWalkKind::kIncludeInlinedFrames), self_(self), exception_handler_(exception_handler), - prev_shadow_frame_(nullptr) { - CHECK(!self_->HasDeoptimizationShadowFrame()); + prev_shadow_frame_(nullptr), + stacked_shadow_frame_pushed_(false) { } bool VisitFrame() OVERRIDE SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { @@ -174,6 +174,13 @@ class DeoptimizeStackVisitor FINAL : public StackVisitor { // This is the upcall, we remember the frame and last pc so that we may long jump to them. exception_handler_->SetHandlerQuickFramePc(GetCurrentQuickFramePc()); exception_handler_->SetHandlerQuickFrame(GetCurrentQuickFrame()); + if (!stacked_shadow_frame_pushed_) { + // In case there is no deoptimized shadow frame for this upcall, we still + // need to push a nullptr to the stack since there is always a matching pop after + // the long jump. + self_->PushStackedShadowFrame(nullptr, kDeoptimizationShadowFrame); + stacked_shadow_frame_pushed_ = true; + } return false; // End stack walk. } else if (method->IsRuntimeMethod()) { // Ignore callee save method. @@ -204,111 +211,115 @@ class DeoptimizeStackVisitor FINAL : public StackVisitor { bool verifier_success = verifier.Verify(); CHECK(verifier_success) << PrettyMethod(m); ShadowFrame* new_frame = ShadowFrame::CreateDeoptimizedFrame(num_regs, nullptr, m, dex_pc); - self_->SetShadowFrameUnderConstruction(new_frame); - const std::vector<int32_t> kinds(verifier.DescribeVRegs(dex_pc)); - - // Markers for dead values, used when the verifier knows a Dex register is undefined, - // or when the compiler knows the register has not been initialized, or is not used - // anymore in the method. - static constexpr uint32_t kDeadValue = 0xEBADDE09; - static constexpr uint64_t kLongDeadValue = 0xEBADDE09EBADDE09; - for (uint16_t reg = 0; reg < num_regs; ++reg) { - VRegKind kind = GetVRegKind(reg, kinds); - switch (kind) { - case kUndefined: - new_frame->SetVReg(reg, kDeadValue); - break; - case kConstant: - new_frame->SetVReg(reg, kinds.at((reg * 2) + 1)); - break; - case kReferenceVReg: { - uint32_t value = 0; - // Check IsReferenceVReg in case the compiled GC map doesn't agree with the verifier. - // We don't want to copy a stale reference into the shadow frame as a reference. - // b/20736048 - if (GetVReg(m, reg, kind, &value) && IsReferenceVReg(m, reg)) { - new_frame->SetVRegReference(reg, reinterpret_cast<mirror::Object*>(value)); - } else { + { + ScopedStackedShadowFramePusher pusher(self_, new_frame, kShadowFrameUnderConstruction); + const std::vector<int32_t> kinds(verifier.DescribeVRegs(dex_pc)); + + // Markers for dead values, used when the verifier knows a Dex register is undefined, + // or when the compiler knows the register has not been initialized, or is not used + // anymore in the method. + static constexpr uint32_t kDeadValue = 0xEBADDE09; + static constexpr uint64_t kLongDeadValue = 0xEBADDE09EBADDE09; + for (uint16_t reg = 0; reg < num_regs; ++reg) { + VRegKind kind = GetVRegKind(reg, kinds); + switch (kind) { + case kUndefined: new_frame->SetVReg(reg, kDeadValue); - } - break; - } - case kLongLoVReg: - if (GetVRegKind(reg + 1, kinds) == kLongHiVReg) { - // Treat it as a "long" register pair. - uint64_t value = 0; - if (GetVRegPair(m, reg, kLongLoVReg, kLongHiVReg, &value)) { - new_frame->SetVRegLong(reg, value); - } else { - new_frame->SetVRegLong(reg, kLongDeadValue); - } - } else { + break; + case kConstant: + new_frame->SetVReg(reg, kinds.at((reg * 2) + 1)); + break; + case kReferenceVReg: { uint32_t value = 0; - if (GetVReg(m, reg, kind, &value)) { - new_frame->SetVReg(reg, value); + // Check IsReferenceVReg in case the compiled GC map doesn't agree with the verifier. + // We don't want to copy a stale reference into the shadow frame as a reference. + // b/20736048 + if (GetVReg(m, reg, kind, &value) && IsReferenceVReg(m, reg)) { + new_frame->SetVRegReference(reg, reinterpret_cast<mirror::Object*>(value)); } else { new_frame->SetVReg(reg, kDeadValue); } + break; } - break; - case kLongHiVReg: - if (GetVRegKind(reg - 1, kinds) == kLongLoVReg) { - // Nothing to do: we treated it as a "long" register pair. - } else { - uint32_t value = 0; - if (GetVReg(m, reg, kind, &value)) { - new_frame->SetVReg(reg, value); + case kLongLoVReg: + if (GetVRegKind(reg + 1, kinds) == kLongHiVReg) { + // Treat it as a "long" register pair. + uint64_t value = 0; + if (GetVRegPair(m, reg, kLongLoVReg, kLongHiVReg, &value)) { + new_frame->SetVRegLong(reg, value); + } else { + new_frame->SetVRegLong(reg, kLongDeadValue); + } } else { - new_frame->SetVReg(reg, kDeadValue); + uint32_t value = 0; + if (GetVReg(m, reg, kind, &value)) { + new_frame->SetVReg(reg, value); + } else { + new_frame->SetVReg(reg, kDeadValue); + } } - } - break; - case kDoubleLoVReg: - if (GetVRegKind(reg + 1, kinds) == kDoubleHiVReg) { - uint64_t value = 0; - if (GetVRegPair(m, reg, kDoubleLoVReg, kDoubleHiVReg, &value)) { - // Treat it as a "double" register pair. - new_frame->SetVRegLong(reg, value); + break; + case kLongHiVReg: + if (GetVRegKind(reg - 1, kinds) == kLongLoVReg) { + // Nothing to do: we treated it as a "long" register pair. } else { - new_frame->SetVRegLong(reg, kLongDeadValue); + uint32_t value = 0; + if (GetVReg(m, reg, kind, &value)) { + new_frame->SetVReg(reg, value); + } else { + new_frame->SetVReg(reg, kDeadValue); + } } - } else { - uint32_t value = 0; - if (GetVReg(m, reg, kind, &value)) { - new_frame->SetVReg(reg, value); + break; + case kDoubleLoVReg: + if (GetVRegKind(reg + 1, kinds) == kDoubleHiVReg) { + uint64_t value = 0; + if (GetVRegPair(m, reg, kDoubleLoVReg, kDoubleHiVReg, &value)) { + // Treat it as a "double" register pair. + new_frame->SetVRegLong(reg, value); + } else { + new_frame->SetVRegLong(reg, kLongDeadValue); + } } else { - new_frame->SetVReg(reg, kDeadValue); + uint32_t value = 0; + if (GetVReg(m, reg, kind, &value)) { + new_frame->SetVReg(reg, value); + } else { + new_frame->SetVReg(reg, kDeadValue); + } } - } - break; - case kDoubleHiVReg: - if (GetVRegKind(reg - 1, kinds) == kDoubleLoVReg) { - // Nothing to do: we treated it as a "double" register pair. - } else { + break; + case kDoubleHiVReg: + if (GetVRegKind(reg - 1, kinds) == kDoubleLoVReg) { + // Nothing to do: we treated it as a "double" register pair. + } else { + uint32_t value = 0; + if (GetVReg(m, reg, kind, &value)) { + new_frame->SetVReg(reg, value); + } else { + new_frame->SetVReg(reg, kDeadValue); + } + } + break; + default: uint32_t value = 0; if (GetVReg(m, reg, kind, &value)) { new_frame->SetVReg(reg, value); } else { new_frame->SetVReg(reg, kDeadValue); } - } - break; - default: - uint32_t value = 0; - if (GetVReg(m, reg, kind, &value)) { - new_frame->SetVReg(reg, value); - } else { - new_frame->SetVReg(reg, kDeadValue); - } - break; + break; + } } } if (prev_shadow_frame_ != nullptr) { prev_shadow_frame_->SetLink(new_frame); } else { - self_->SetDeoptimizationShadowFrame(new_frame); + // Will be popped after the long jump after DeoptimizeStack(), + // right before interpreter::EnterInterpreterFromDeoptimize(). + stacked_shadow_frame_pushed_ = true; + self_->PushStackedShadowFrame(new_frame, kDeoptimizationShadowFrame); } - self_->ClearShadowFrameUnderConstruction(); prev_shadow_frame_ = new_frame; return true; } @@ -316,6 +327,7 @@ class DeoptimizeStackVisitor FINAL : public StackVisitor { Thread* const self_; QuickExceptionHandler* const exception_handler_; ShadowFrame* prev_shadow_frame_; + bool stacked_shadow_frame_pushed_; DISALLOW_COPY_AND_ASSIGN(DeoptimizeStackVisitor); }; diff --git a/runtime/thread.cc b/runtime/thread.cc index 4203b96f24..3f49ab9e8c 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -147,29 +147,38 @@ void Thread::ResetQuickAllocEntryPointsForThread() { ResetQuickAllocEntryPoints(&tlsPtr_.quick_entrypoints); } -void Thread::SetDeoptimizationShadowFrame(ShadowFrame* sf) { - tlsPtr_.deoptimization_shadow_frame = sf; +void Thread::PushAndClearDeoptimizationReturnValue() { + DeoptimizationReturnValueRecord* record = new DeoptimizationReturnValueRecord( + tls64_.deoptimization_return_value, + tls32_.deoptimization_return_value_is_reference, + tlsPtr_.deoptimization_return_value_stack); + tlsPtr_.deoptimization_return_value_stack = record; + ClearDeoptimizationReturnValue(); } -void Thread::SetDeoptimizationReturnValue(const JValue& ret_val) { - tls64_.deoptimization_return_value.SetJ(ret_val.GetJ()); +JValue Thread::PopDeoptimizationReturnValue() { + DeoptimizationReturnValueRecord* record = tlsPtr_.deoptimization_return_value_stack; + DCHECK(record != nullptr); + tlsPtr_.deoptimization_return_value_stack = record->GetLink(); + JValue ret_val(record->GetReturnValue()); + delete record; + return ret_val; } -ShadowFrame* Thread::GetAndClearDeoptimizationShadowFrame(JValue* ret_val) { - ShadowFrame* sf = tlsPtr_.deoptimization_shadow_frame; - tlsPtr_.deoptimization_shadow_frame = nullptr; - ret_val->SetJ(tls64_.deoptimization_return_value.GetJ()); - return sf; +void Thread::PushStackedShadowFrame(ShadowFrame* sf, StackedShadowFrameType type) { + StackedShadowFrameRecord* record = new StackedShadowFrameRecord( + sf, type, tlsPtr_.stacked_shadow_frame_record); + tlsPtr_.stacked_shadow_frame_record = record; } -void Thread::SetShadowFrameUnderConstruction(ShadowFrame* sf) { - sf->SetLink(tlsPtr_.shadow_frame_under_construction); - tlsPtr_.shadow_frame_under_construction = sf; -} - -void Thread::ClearShadowFrameUnderConstruction() { - CHECK_NE(static_cast<ShadowFrame*>(nullptr), tlsPtr_.shadow_frame_under_construction); - tlsPtr_.shadow_frame_under_construction = tlsPtr_.shadow_frame_under_construction->GetLink(); +ShadowFrame* Thread::PopStackedShadowFrame(StackedShadowFrameType type) { + StackedShadowFrameRecord* record = tlsPtr_.stacked_shadow_frame_record; + DCHECK(record != nullptr); + DCHECK(record->GetType() == type); + tlsPtr_.stacked_shadow_frame_record = record->GetLink(); + ShadowFrame* shadow_frame = record->GetShadowFrame(); + delete record; + return shadow_frame; } void Thread::InitTid() { @@ -2387,21 +2396,27 @@ void Thread::VisitRoots(RootVisitor* visitor) { if (tlsPtr_.debug_invoke_req != nullptr) { tlsPtr_.debug_invoke_req->VisitRoots(visitor, RootInfo(kRootDebugger, thread_id)); } - if (tlsPtr_.deoptimization_shadow_frame != nullptr) { + if (tlsPtr_.stacked_shadow_frame_record != nullptr) { RootCallbackVisitor visitor_to_callback(visitor, thread_id); ReferenceMapVisitor<RootCallbackVisitor> mapper(this, nullptr, visitor_to_callback); - for (ShadowFrame* shadow_frame = tlsPtr_.deoptimization_shadow_frame; shadow_frame != nullptr; - shadow_frame = shadow_frame->GetLink()) { - mapper.VisitShadowFrame(shadow_frame); + for (StackedShadowFrameRecord* record = tlsPtr_.stacked_shadow_frame_record; + record != nullptr; + record = record->GetLink()) { + for (ShadowFrame* shadow_frame = record->GetShadowFrame(); + shadow_frame != nullptr; + shadow_frame = shadow_frame->GetLink()) { + mapper.VisitShadowFrame(shadow_frame); + } } } - if (tlsPtr_.shadow_frame_under_construction != nullptr) { - RootCallbackVisitor visitor_to_callback(visitor, thread_id); - ReferenceMapVisitor<RootCallbackVisitor> mapper(this, nullptr, visitor_to_callback); - for (ShadowFrame* shadow_frame = tlsPtr_.shadow_frame_under_construction; - shadow_frame != nullptr; - shadow_frame = shadow_frame->GetLink()) { - mapper.VisitShadowFrame(shadow_frame); + if (tlsPtr_.deoptimization_return_value_stack != nullptr) { + for (DeoptimizationReturnValueRecord* record = tlsPtr_.deoptimization_return_value_stack; + record != nullptr; + record = record->GetLink()) { + if (record->IsReference()) { + visitor->VisitRootIfNonNull(record->GetGCRoot(), + RootInfo(kRootThreadObject, thread_id)); + } } } for (auto* verifier = tlsPtr_.method_verifier; verifier != nullptr; verifier = verifier->link_) { diff --git a/runtime/thread.h b/runtime/thread.h index 3f0d0a59a9..e996fc96af 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -99,6 +99,55 @@ enum ThreadFlag { kCheckpointRequest = 2 // Request that the thread do some checkpoint work and then continue. }; +enum StackedShadowFrameType { + kShadowFrameUnderConstruction, + kDeoptimizationShadowFrame +}; + +class StackedShadowFrameRecord { + public: + StackedShadowFrameRecord(ShadowFrame* shadow_frame, + StackedShadowFrameType type, + StackedShadowFrameRecord* link) + : shadow_frame_(shadow_frame), + type_(type), + link_(link) {} + + ShadowFrame* GetShadowFrame() const { return shadow_frame_; } + bool GetType() const { return type_; } + StackedShadowFrameRecord* GetLink() const { return link_; } + + private: + ShadowFrame* const shadow_frame_; + const StackedShadowFrameType type_; + StackedShadowFrameRecord* const link_; + + DISALLOW_COPY_AND_ASSIGN(StackedShadowFrameRecord); +}; + +class DeoptimizationReturnValueRecord { + public: + DeoptimizationReturnValueRecord(const JValue& ret_val, + bool is_reference, + DeoptimizationReturnValueRecord* link) + : ret_val_(ret_val), is_reference_(is_reference), link_(link) {} + + JValue GetReturnValue() const { return ret_val_; } + bool IsReference() const { return is_reference_; } + DeoptimizationReturnValueRecord* GetLink() const { return link_; } + mirror::Object** GetGCRoot() { + DCHECK(is_reference_); + return ret_val_.GetGCRoot(); + } + + private: + JValue ret_val_; + const bool is_reference_; + DeoptimizationReturnValueRecord* const link_; + + DISALLOW_COPY_AND_ASSIGN(DeoptimizationReturnValueRecord); +}; + static constexpr size_t kNumRosAllocThreadLocalSizeBrackets = 34; // Thread's stack layout for implicit stack overflow checks: @@ -790,21 +839,25 @@ class Thread { return reinterpret_cast<mirror::Throwable*>(-1); } - void SetDeoptimizationShadowFrame(ShadowFrame* sf); - void SetDeoptimizationReturnValue(const JValue& ret_val); - - ShadowFrame* GetAndClearDeoptimizationShadowFrame(JValue* ret_val); - - bool HasDeoptimizationShadowFrame() const { - return tlsPtr_.deoptimization_shadow_frame != nullptr; + // Currently deoptimization invokes verifier which can trigger class loading + // and execute Java code, so there might be nested deoptimizations happening. + // We need to save the ongoing deoptimization shadow frames and return + // values on stacks. + void SetDeoptimizationReturnValue(const JValue& ret_val, bool is_reference) { + tls64_.deoptimization_return_value.SetJ(ret_val.GetJ()); + tls32_.deoptimization_return_value_is_reference = is_reference; } - - void SetShadowFrameUnderConstruction(ShadowFrame* sf); - void ClearShadowFrameUnderConstruction(); - - bool HasShadowFrameUnderConstruction() const { - return tlsPtr_.shadow_frame_under_construction != nullptr; + bool IsDeoptimizationReturnValueReference() { + return tls32_.deoptimization_return_value_is_reference; + } + void ClearDeoptimizationReturnValue() { + tls64_.deoptimization_return_value.SetJ(0); + tls32_.deoptimization_return_value_is_reference = false; } + void PushAndClearDeoptimizationReturnValue(); + JValue PopDeoptimizationReturnValue(); + void PushStackedShadowFrame(ShadowFrame* sf, StackedShadowFrameType type); + ShadowFrame* PopStackedShadowFrame(StackedShadowFrameType type); std::deque<instrumentation::InstrumentationStackFrame>* GetInstrumentationStack() { return tlsPtr_.instrumentation_stack; @@ -1048,7 +1101,8 @@ class Thread { explicit tls_32bit_sized_values(bool is_daemon) : suspend_count(0), debug_suspend_count(0), thin_lock_thread_id(0), tid(0), daemon(is_daemon), throwing_OutOfMemoryError(false), no_thread_suspension(0), - thread_exit_check_count(0), handling_signal_(false), suspended_at_suspend_check(false), + thread_exit_check_count(0), handling_signal_(false), + deoptimization_return_value_is_reference(false), suspended_at_suspend_check(false), ready_for_debug_invoke(false), debug_method_entry_(false) { } @@ -1089,6 +1143,10 @@ class Thread { // True if signal is being handled by this thread. bool32_t handling_signal_; + // True if the return value for interpreter after deoptimization is a reference. + // For gc purpose. + bool32_t deoptimization_return_value_is_reference; + // True if the thread is suspended in FullSuspendCheck(). This is // used to distinguish runnable threads that are suspended due to // a normal suspend check from other threads. @@ -1124,8 +1182,9 @@ class Thread { stack_trace_sample(nullptr), wait_next(nullptr), monitor_enter_object(nullptr), top_handle_scope(nullptr), class_loader_override(nullptr), long_jump_context(nullptr), instrumentation_stack(nullptr), debug_invoke_req(nullptr), single_step_control(nullptr), - deoptimization_shadow_frame(nullptr), shadow_frame_under_construction(nullptr), name(nullptr), - pthread_self(0), last_no_thread_suspension_cause(nullptr), thread_local_start(nullptr), + stacked_shadow_frame_record(nullptr), deoptimization_return_value_stack(nullptr), + name(nullptr), pthread_self(0), + last_no_thread_suspension_cause(nullptr), thread_local_start(nullptr), thread_local_pos(nullptr), thread_local_end(nullptr), thread_local_objects(0), thread_local_alloc_stack_top(nullptr), thread_local_alloc_stack_end(nullptr), nested_signal_state(nullptr), flip_function(nullptr), method_verifier(nullptr) { @@ -1201,11 +1260,13 @@ class Thread { // JDWP single-stepping support. SingleStepControl* single_step_control; - // Shadow frame stack that is used temporarily during the deoptimization of a method. - ShadowFrame* deoptimization_shadow_frame; + // For gc purpose, a shadow frame record stack that keeps track of: + // 1) shadow frames under construction. + // 2) deoptimization shadow frames. + StackedShadowFrameRecord* stacked_shadow_frame_record; - // Shadow frame stack that is currently under construction but not yet on the stack - ShadowFrame* shadow_frame_under_construction; + // Deoptimization return value record stack. + DeoptimizationReturnValueRecord* deoptimization_return_value_stack; // A cached copy of the java.lang.Thread's name. std::string* name; @@ -1293,6 +1354,23 @@ class ScopedAssertNoThreadSuspension { const char* const old_cause_; }; +class ScopedStackedShadowFramePusher { + public: + ScopedStackedShadowFramePusher(Thread* self, ShadowFrame* sf, StackedShadowFrameType type) + : self_(self), type_(type) { + self_->PushStackedShadowFrame(sf, type); + } + ~ScopedStackedShadowFramePusher() { + self_->PopStackedShadowFrame(type_); + } + + private: + Thread* const self_; + const StackedShadowFrameType type_; + + DISALLOW_COPY_AND_ASSIGN(ScopedStackedShadowFramePusher); +}; + std::ostream& operator<<(std::ostream& os, const Thread& thread); } // namespace art diff --git a/test/441-checker-inliner/src/Main.java b/test/441-checker-inliner/src/Main.java index df969a488e..3899d7fb26 100644 --- a/test/441-checker-inliner/src/Main.java +++ b/test/441-checker-inliner/src/Main.java @@ -19,7 +19,7 @@ public class Main { /// CHECK-START: void Main.InlineVoid() inliner (before) /// CHECK-DAG: <<Const42:i\d+>> IntConstant 42 /// CHECK-DAG: InvokeStaticOrDirect - /// CHECK-DAG: InvokeStaticOrDirect [<<Const42>>] + /// CHECK-DAG: InvokeStaticOrDirect [<<Const42>>,{{[ij]\d+}}] /// CHECK-START: void Main.InlineVoid() inliner (after) /// CHECK-NOT: InvokeStaticOrDirect @@ -31,7 +31,7 @@ public class Main { /// CHECK-START: int Main.InlineParameter(int) inliner (before) /// CHECK-DAG: <<Param:i\d+>> ParameterValue - /// CHECK-DAG: <<Result:i\d+>> InvokeStaticOrDirect [<<Param>>] + /// CHECK-DAG: <<Result:i\d+>> InvokeStaticOrDirect [<<Param>>,{{[ij]\d+}}] /// CHECK-DAG: Return [<<Result>>] /// CHECK-START: int Main.InlineParameter(int) inliner (after) @@ -44,7 +44,7 @@ public class Main { /// CHECK-START: long Main.InlineWideParameter(long) inliner (before) /// CHECK-DAG: <<Param:j\d+>> ParameterValue - /// CHECK-DAG: <<Result:j\d+>> InvokeStaticOrDirect [<<Param>>] + /// CHECK-DAG: <<Result:j\d+>> InvokeStaticOrDirect [<<Param>>,{{[ij]\d+}}] /// CHECK-DAG: Return [<<Result>>] /// CHECK-START: long Main.InlineWideParameter(long) inliner (after) @@ -57,7 +57,7 @@ public class Main { /// CHECK-START: java.lang.Object Main.InlineReferenceParameter(java.lang.Object) inliner (before) /// CHECK-DAG: <<Param:l\d+>> ParameterValue - /// CHECK-DAG: <<Result:l\d+>> InvokeStaticOrDirect [<<Param>>] + /// CHECK-DAG: <<Result:l\d+>> InvokeStaticOrDirect [<<Param>>,{{[ij]\d+}}] /// CHECK-DAG: Return [<<Result>>] /// CHECK-START: java.lang.Object Main.InlineReferenceParameter(java.lang.Object) inliner (after) @@ -130,8 +130,8 @@ public class Main { /// CHECK-DAG: <<Const1:i\d+>> IntConstant 1 /// CHECK-DAG: <<Const3:i\d+>> IntConstant 3 /// CHECK-DAG: <<Const5:i\d+>> IntConstant 5 - /// CHECK-DAG: <<Add:i\d+>> InvokeStaticOrDirect [<<Const1>>,<<Const3>>] - /// CHECK-DAG: <<Sub:i\d+>> InvokeStaticOrDirect [<<Const5>>,<<Const3>>] + /// CHECK-DAG: <<Add:i\d+>> InvokeStaticOrDirect [<<Const1>>,<<Const3>>,{{[ij]\d+}}] + /// CHECK-DAG: <<Sub:i\d+>> InvokeStaticOrDirect [<<Const5>>,<<Const3>>,{{[ij]\d+}}] /// CHECK-DAG: <<Phi:i\d+>> Phi [<<Add>>,<<Sub>>] /// CHECK-DAG: Return [<<Phi>>] diff --git a/test/478-checker-clinit-check-pruning/src/Main.java b/test/478-checker-clinit-check-pruning/src/Main.java index a2c98c9363..e6aab630f9 100644 --- a/test/478-checker-clinit-check-pruning/src/Main.java +++ b/test/478-checker-clinit-check-pruning/src/Main.java @@ -26,7 +26,7 @@ public class Main { /// CHECK-START: void Main.invokeStaticInlined() builder (after) /// CHECK-DAG: <<LoadClass:l\d+>> LoadClass gen_clinit_check:false /// CHECK-DAG: <<ClinitCheck:l\d+>> ClinitCheck [<<LoadClass>>] - /// CHECK-DAG: InvokeStaticOrDirect [<<ClinitCheck>>] + /// CHECK-DAG: InvokeStaticOrDirect [{{[ij]\d+}},<<ClinitCheck>>] /// CHECK-START: void Main.invokeStaticInlined() inliner (after) /// CHECK-DAG: <<LoadClass:l\d+>> LoadClass gen_clinit_check:false @@ -69,12 +69,12 @@ public class Main { /// CHECK-START: void Main.invokeStaticNotInlined() builder (after) /// CHECK-DAG: <<LoadClass:l\d+>> LoadClass gen_clinit_check:false /// CHECK-DAG: <<ClinitCheck:l\d+>> ClinitCheck [<<LoadClass>>] - /// CHECK-DAG: InvokeStaticOrDirect [<<ClinitCheck>>] + /// CHECK-DAG: InvokeStaticOrDirect [{{[ij]\d+}},<<ClinitCheck>>] /// CHECK-START: void Main.invokeStaticNotInlined() inliner (after) /// CHECK-DAG: <<LoadClass:l\d+>> LoadClass gen_clinit_check:false /// CHECK-DAG: <<ClinitCheck:l\d+>> ClinitCheck [<<LoadClass>>] - /// CHECK-DAG: InvokeStaticOrDirect [<<ClinitCheck>>] + /// CHECK-DAG: InvokeStaticOrDirect [{{[ij]\d+}},<<ClinitCheck>>] // The following checks ensure the clinit check and load class // instructions added by the builder are pruned by the diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index 3ca1c065e5..e95f147e65 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -473,11 +473,6 @@ endif TEST_ART_BROKEN_HEAP_POISONING_RUN_TESTS := -# Test 137-cfi works in 32-bit only until we enable 64-bit ELF files. -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),137-cfi,64) - # Clear variables ahead of appending to them when defining tests. $(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=)) $(foreach target, $(TARGET_TYPES), \ |