diff options
Diffstat (limited to 'compiler/optimizing')
28 files changed, 1922 insertions, 376 deletions
diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc index 9c2facb75e..955deaa0ae 100644 --- a/compiler/optimizing/builder.cc +++ b/compiler/optimizing/builder.cc @@ -161,7 +161,7 @@ void HGraphBuilder::InitializeParameters(uint16_t number_of_parameters) { if (!dex_compilation_unit_->IsStatic()) { // Add the implicit 'this' argument, not expressed in the signature. HParameterValue* parameter = - new (arena_) HParameterValue(parameter_index++, Primitive::kPrimNot); + new (arena_) HParameterValue(parameter_index++, Primitive::kPrimNot, true); entry_block_->AddInstruction(parameter); HLocal* local = GetLocalAt(locals_index++); entry_block_->AddInstruction(new (arena_) HStoreLocal(local, parameter)); @@ -613,9 +613,12 @@ bool HGraphBuilder::BuildInvoke(const Instruction& instruction, // Sharpening to kDirect only works if we compile PIC. DCHECK((optimized_invoke_type == invoke_type) || (optimized_invoke_type != kDirect) || compiler_driver_->GetCompilerOptions().GetCompilePic()); + bool is_recursive = + (target_method.dex_method_index == outer_compilation_unit_->GetDexMethodIndex()); + DCHECK(!is_recursive || (target_method.dex_file == outer_compilation_unit_->GetDexFile())); invoke = new (arena_) HInvokeStaticOrDirect( arena_, number_of_arguments, return_type, dex_pc, target_method.dex_method_index, - optimized_invoke_type); + is_recursive, optimized_invoke_type); } size_t start_index = 0; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 0fe28e8352..7731a10b17 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -527,6 +527,8 @@ void CodeGeneratorARM::GenerateFrameEntry() { bool skip_overflow_check = IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); + __ Bind(&frame_entry_label_); + if (!skip_overflow_check) { __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); __ LoadFromOffset(kLoadWord, IP, IP, 0); @@ -1185,18 +1187,22 @@ void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirec // temp = method; codegen_->LoadCurrentMethod(temp); - // temp = temp->dex_cache_resolved_methods_; - __ LoadFromOffset( - kLoadWord, temp, temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()); - // temp = temp[index_in_cache] - __ LoadFromOffset( - kLoadWord, temp, temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())); - // LR = temp[offset_of_quick_compiled_code] - __ LoadFromOffset(kLoadWord, LR, temp, - mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArmWordSize).Int32Value()); - // LR() - __ blx(LR); + if (!invoke->IsRecursive()) { + // temp = temp->dex_cache_resolved_methods_; + __ LoadFromOffset( + kLoadWord, temp, temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value()); + // temp = temp[index_in_cache] + __ LoadFromOffset( + kLoadWord, temp, temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex())); + // LR = temp[offset_of_quick_compiled_code] + __ LoadFromOffset(kLoadWord, LR, temp, + mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kArmWordSize).Int32Value()); + // LR() + __ blx(LR); + } else { + __ bl(codegen_->GetFrameEntryLabel()); + } codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); DCHECK(!codegen_->IsLeafMethod()); diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index dd69e4dd9c..4b03dffd38 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -247,9 +247,12 @@ class CodeGeneratorARM : public CodeGenerator { void ComputeSpillMask() OVERRIDE; + Label* GetFrameEntryLabel() { return &frame_entry_label_; } + private: // Labels for each block that will be compiled. GrowableArray<Label> block_labels_; + Label frame_entry_label_; LocationsBuilderARM location_builder_; InstructionCodeGeneratorARM instruction_visitor_; ParallelMoveResolverARM move_resolver_; diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 1f561b725a..78ae55ea1e 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -16,9 +16,12 @@ #include "code_generator_arm64.h" +#include "common_arm64.h" #include "entrypoints/quick/quick_entrypoints.h" #include "entrypoints/quick/quick_entrypoints_enum.h" #include "gc/accounting/card_table.h" +#include "intrinsics.h" +#include "intrinsics_arm64.h" #include "mirror/array-inl.h" #include "mirror/art_method.h" #include "mirror/class.h" @@ -35,175 +38,34 @@ using namespace vixl; // NOLINT(build/namespaces) #error "ARM64 Codegen VIXL macro-assembler macro already defined." #endif - namespace art { namespace arm64 { -// TODO: Tune the use of Load-Acquire, Store-Release vs Data Memory Barriers. -// For now we prefer the use of load-acquire, store-release over explicit memory barriers. -static constexpr bool kUseAcquireRelease = true; +using helpers::CPURegisterFrom; +using helpers::DRegisterFrom; +using helpers::FPRegisterFrom; +using helpers::HeapOperand; +using helpers::HeapOperandFrom; +using helpers::InputCPURegisterAt; +using helpers::InputFPRegisterAt; +using helpers::InputRegisterAt; +using helpers::InputOperandAt; +using helpers::Int64ConstantFrom; +using helpers::LocationFrom; +using helpers::OperandFromMemOperand; +using helpers::OutputCPURegister; +using helpers::OutputFPRegister; +using helpers::OutputRegister; +using helpers::RegisterFrom; +using helpers::StackOperandFrom; +using helpers::VIXLRegCodeFromART; +using helpers::WRegisterFrom; +using helpers::XRegisterFrom; + static constexpr size_t kHeapRefSize = sizeof(mirror::HeapReference<mirror::Object>); static constexpr int kCurrentMethodStackOffset = 0; -namespace { - -bool IsFPType(Primitive::Type type) { - return type == Primitive::kPrimFloat || type == Primitive::kPrimDouble; -} - -bool IsIntegralType(Primitive::Type type) { - switch (type) { - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimLong: - return true; - default: - return false; - } -} - -bool Is64BitType(Primitive::Type type) { - return type == Primitive::kPrimLong || type == Primitive::kPrimDouble; -} - -// Convenience helpers to ease conversion to and from VIXL operands. -static_assert((SP == 31) && (WSP == 31) && (XZR == 32) && (WZR == 32), - "Unexpected values for register codes."); - -int VIXLRegCodeFromART(int code) { - if (code == SP) { - return vixl::kSPRegInternalCode; - } - if (code == XZR) { - return vixl::kZeroRegCode; - } - return code; -} - -int ARTRegCodeFromVIXL(int code) { - if (code == vixl::kSPRegInternalCode) { - return SP; - } - if (code == vixl::kZeroRegCode) { - return XZR; - } - return code; -} - -Register XRegisterFrom(Location location) { - DCHECK(location.IsRegister()); - return Register::XRegFromCode(VIXLRegCodeFromART(location.reg())); -} - -Register WRegisterFrom(Location location) { - DCHECK(location.IsRegister()); - return Register::WRegFromCode(VIXLRegCodeFromART(location.reg())); -} - -Register RegisterFrom(Location location, Primitive::Type type) { - DCHECK(type != Primitive::kPrimVoid && !IsFPType(type)); - return type == Primitive::kPrimLong ? XRegisterFrom(location) : WRegisterFrom(location); -} - -Register OutputRegister(HInstruction* instr) { - return RegisterFrom(instr->GetLocations()->Out(), instr->GetType()); -} - -Register InputRegisterAt(HInstruction* instr, int input_index) { - return RegisterFrom(instr->GetLocations()->InAt(input_index), - instr->InputAt(input_index)->GetType()); -} - -FPRegister DRegisterFrom(Location location) { - DCHECK(location.IsFpuRegister()); - return FPRegister::DRegFromCode(location.reg()); -} - -FPRegister SRegisterFrom(Location location) { - DCHECK(location.IsFpuRegister()); - return FPRegister::SRegFromCode(location.reg()); -} - -FPRegister FPRegisterFrom(Location location, Primitive::Type type) { - DCHECK(IsFPType(type)); - return type == Primitive::kPrimDouble ? DRegisterFrom(location) : SRegisterFrom(location); -} - -FPRegister OutputFPRegister(HInstruction* instr) { - return FPRegisterFrom(instr->GetLocations()->Out(), instr->GetType()); -} - -FPRegister InputFPRegisterAt(HInstruction* instr, int input_index) { - return FPRegisterFrom(instr->GetLocations()->InAt(input_index), - instr->InputAt(input_index)->GetType()); -} - -CPURegister CPURegisterFrom(Location location, Primitive::Type type) { - return IsFPType(type) ? CPURegister(FPRegisterFrom(location, type)) - : CPURegister(RegisterFrom(location, type)); -} - -CPURegister OutputCPURegister(HInstruction* instr) { - return IsFPType(instr->GetType()) ? static_cast<CPURegister>(OutputFPRegister(instr)) - : static_cast<CPURegister>(OutputRegister(instr)); -} - -CPURegister InputCPURegisterAt(HInstruction* instr, int index) { - return IsFPType(instr->InputAt(index)->GetType()) - ? static_cast<CPURegister>(InputFPRegisterAt(instr, index)) - : static_cast<CPURegister>(InputRegisterAt(instr, index)); -} - -int64_t Int64ConstantFrom(Location location) { - HConstant* instr = location.GetConstant(); - return instr->IsIntConstant() ? instr->AsIntConstant()->GetValue() - : instr->AsLongConstant()->GetValue(); -} - -Operand OperandFrom(Location location, Primitive::Type type) { - if (location.IsRegister()) { - return Operand(RegisterFrom(location, type)); - } else { - return Operand(Int64ConstantFrom(location)); - } -} - -Operand InputOperandAt(HInstruction* instr, int input_index) { - return OperandFrom(instr->GetLocations()->InAt(input_index), - instr->InputAt(input_index)->GetType()); -} - -MemOperand StackOperandFrom(Location location) { - return MemOperand(sp, location.GetStackIndex()); -} - -MemOperand HeapOperand(const Register& base, size_t offset = 0) { - // A heap reference must be 32bit, so fit in a W register. - DCHECK(base.IsW()); - return MemOperand(base.X(), offset); -} - -MemOperand HeapOperand(const Register& base, Offset offset) { - return HeapOperand(base, offset.SizeValue()); -} - -MemOperand HeapOperandFrom(Location location, Offset offset) { - return HeapOperand(RegisterFrom(location, Primitive::kPrimNot), offset); -} - -Location LocationFrom(const Register& reg) { - return Location::RegisterLocation(ARTRegCodeFromVIXL(reg.code())); -} - -Location LocationFrom(const FPRegister& fpreg) { - return Location::FpuRegisterLocation(fpreg.code()); -} - -} // namespace - inline Condition ARM64Condition(IfCondition cond) { switch (cond) { case kCondEQ: return eq; @@ -264,20 +126,6 @@ Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type retur #define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()-> #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, x).Int32Value() -class SlowPathCodeARM64 : public SlowPathCode { - public: - SlowPathCodeARM64() : entry_label_(), exit_label_() {} - - vixl::Label* GetEntryLabel() { return &entry_label_; } - vixl::Label* GetExitLabel() { return &exit_label_; } - - private: - vixl::Label entry_label_; - vixl::Label exit_label_; - - DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64); -}; - class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 { public: BoundsCheckSlowPathARM64(HBoundsCheck* instruction, @@ -532,18 +380,20 @@ Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type LOG(FATAL) << "Unreachable type " << type; } - if (IsFPType(type) && (fp_index_ < calling_convention.GetNumberOfFpuRegisters())) { + if (Primitive::IsFloatingPointType(type) && + (fp_index_ < calling_convention.GetNumberOfFpuRegisters())) { next_location = LocationFrom(calling_convention.GetFpuRegisterAt(fp_index_++)); - } else if (!IsFPType(type) && (gp_index_ < calling_convention.GetNumberOfRegisters())) { + } else if (!Primitive::IsFloatingPointType(type) && + (gp_index_ < calling_convention.GetNumberOfRegisters())) { next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_++)); } else { size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_); - next_location = Is64BitType(type) ? Location::DoubleStackSlot(stack_offset) - : Location::StackSlot(stack_offset); + next_location = Primitive::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset) + : Location::StackSlot(stack_offset); } // Space on the stack is reserved for all arguments. - stack_index_ += Is64BitType(type) ? 2 : 1; + stack_index_ += Primitive::Is64BitType(type) ? 2 : 1; return next_location; } @@ -591,6 +441,8 @@ void ParallelMoveResolverARM64::SpillScratch(int reg) { } void CodeGeneratorARM64::GenerateFrameEntry() { + __ Bind(&frame_entry_label_); + bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod(); if (do_overflow_check) { UseScratchRegisterScope temps(GetVIXLAssembler()); @@ -602,7 +454,7 @@ void CodeGeneratorARM64::GenerateFrameEntry() { } int frame_size = GetFrameSize(); - __ Str(w0, MemOperand(sp, -frame_size, PreIndex)); + __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex)); __ PokeCPURegList(GetFramePreservedRegisters(), frame_size - FrameEntrySpillSize()); // Stack layout: @@ -654,7 +506,7 @@ void CodeGeneratorARM64::Move(HInstruction* instruction, MoveLocation(location, temp_location, type); } else if (instruction->IsLoadLocal()) { uint32_t stack_slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal()); - if (Is64BitType(type)) { + if (Primitive::Is64BitType(type)) { MoveLocation(location, Location::DoubleStackSlot(stack_slot), type); } else { MoveLocation(location, Location::StackSlot(stack_slot), type); @@ -730,7 +582,7 @@ Location CodeGeneratorARM64::AllocateFreeRegister(Primitive::Type type) const { LOG(FATAL) << "Unreachable type " << type; } - if (IsFPType(type)) { + if (Primitive::IsFloatingPointType(type)) { ssize_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfAllocatableFPRegisters); DCHECK_NE(reg, -1); return Location::FpuRegisterLocation(reg); @@ -822,8 +674,8 @@ void CodeGeneratorARM64::MoveLocation(Location destination, Location source, Pri type = destination.IsRegister() ? Primitive::kPrimLong : Primitive::kPrimDouble; } } - DCHECK((destination.IsFpuRegister() && IsFPType(type)) || - (destination.IsRegister() && !IsFPType(type))); + DCHECK((destination.IsFpuRegister() && Primitive::IsFloatingPointType(type)) || + (destination.IsRegister() && !Primitive::IsFloatingPointType(type))); CPURegister dst = CPURegisterFrom(destination, type); if (source.IsStackSlot() || source.IsDoubleStackSlot()) { DCHECK(dst.Is64Bits() == source.IsDoubleStackSlot()); @@ -849,8 +701,8 @@ void CodeGeneratorARM64::MoveLocation(Location destination, Location source, Pri type = destination.IsStackSlot() ? Primitive::kPrimFloat : Primitive::kPrimDouble; } } - DCHECK((destination.IsDoubleStackSlot() == Is64BitType(type)) && - (source.IsFpuRegister() == IsFPType(type))); + DCHECK((destination.IsDoubleStackSlot() == Primitive::Is64BitType(type)) && + (source.IsFpuRegister() == Primitive::IsFloatingPointType(type))); __ Str(CPURegisterFrom(source, type), StackOperandFrom(destination)); } else if (source.IsConstant()) { DCHECK(unspecified_type || CoherentConstantAndType(source, type)); @@ -963,7 +815,7 @@ void CodeGeneratorARM64::Load(Primitive::Type type, case Primitive::kPrimLong: case Primitive::kPrimFloat: case Primitive::kPrimDouble: - DCHECK_EQ(dst.Is64Bits(), Is64BitType(type)); + DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); __ Ldr(dst, src); break; case Primitive::kPrimVoid: @@ -978,12 +830,11 @@ void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction, Register temp_base = temps.AcquireX(); Primitive::Type type = instruction->GetType(); - DCHECK(!src.IsRegisterOffset()); DCHECK(!src.IsPreIndex()); DCHECK(!src.IsPostIndex()); // TODO(vixl): Let the MacroAssembler handle MemOperand. - __ Add(temp_base, src.base(), src.offset()); + __ Add(temp_base, src.base(), OperandFromMemOperand(src)); MemOperand base = MemOperand(temp_base); switch (type) { case Primitive::kPrimBoolean: @@ -1007,14 +858,14 @@ void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction, case Primitive::kPrimInt: case Primitive::kPrimNot: case Primitive::kPrimLong: - DCHECK_EQ(dst.Is64Bits(), Is64BitType(type)); + DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); __ Ldar(Register(dst), base); MaybeRecordImplicitNullCheck(instruction); break; case Primitive::kPrimFloat: case Primitive::kPrimDouble: { DCHECK(dst.IsFPRegister()); - DCHECK_EQ(dst.Is64Bits(), Is64BitType(type)); + DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW(); __ Ldar(temp, base); @@ -1044,7 +895,7 @@ void CodeGeneratorARM64::Store(Primitive::Type type, case Primitive::kPrimLong: case Primitive::kPrimFloat: case Primitive::kPrimDouble: - DCHECK_EQ(src.Is64Bits(), Is64BitType(type)); + DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type)); __ Str(src, dst); break; case Primitive::kPrimVoid: @@ -1058,12 +909,12 @@ void CodeGeneratorARM64::StoreRelease(Primitive::Type type, UseScratchRegisterScope temps(GetVIXLAssembler()); Register temp_base = temps.AcquireX(); - DCHECK(!dst.IsRegisterOffset()); DCHECK(!dst.IsPreIndex()); DCHECK(!dst.IsPostIndex()); // TODO(vixl): Let the MacroAssembler handle this. - __ Add(temp_base, dst.base(), dst.offset()); + Operand op = OperandFromMemOperand(dst); + __ Add(temp_base, dst.base(), op); MemOperand base = MemOperand(temp_base); switch (type) { case Primitive::kPrimBoolean: @@ -1077,13 +928,13 @@ void CodeGeneratorARM64::StoreRelease(Primitive::Type type, case Primitive::kPrimInt: case Primitive::kPrimNot: case Primitive::kPrimLong: - DCHECK_EQ(src.Is64Bits(), Is64BitType(type)); + DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type)); __ Stlr(Register(src), base); break; case Primitive::kPrimFloat: case Primitive::kPrimDouble: { DCHECK(src.IsFPRegister()); - DCHECK_EQ(src.Is64Bits(), Is64BitType(type)); + DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type)); Register temp = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW(); __ Fmov(temp, FPRegister(src)); @@ -1956,19 +1807,37 @@ void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invok } void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { + IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena()); + if (intrinsic.TryDispatch(invoke)) { + return; + } + HandleInvoke(invoke); } void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { + IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena()); + if (intrinsic.TryDispatch(invoke)) { + return; + } + HandleInvoke(invoke); } -void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { - Register temp = WRegisterFrom(invoke->GetLocations()->GetTemp(0)); - // Make sure that ArtMethod* is passed in W0 as per the calling convention - DCHECK(temp.Is(w0)); +static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codegen) { + if (invoke->GetLocations()->Intrinsified()) { + IntrinsicCodeGeneratorARM64 intrinsic(codegen); + intrinsic.Dispatch(invoke); + return true; + } + return false; +} + +void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Register temp) { + // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention. + DCHECK(temp.Is(kArtMethodRegister)); size_t index_in_cache = mirror::Array::DataOffset(kHeapRefSize).SizeValue() + - invoke->GetDexMethodIndex() * kHeapRefSize; + invoke->GetDexMethodIndex() * kHeapRefSize; // TODO: Implement all kinds of calls: // 1) boot -> boot @@ -1978,22 +1847,39 @@ void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDir // Currently we implement the app -> app logic, which looks up in the resolve cache. // temp = method; - codegen_->LoadCurrentMethod(temp); - // temp = temp->dex_cache_resolved_methods_; - __ Ldr(temp, HeapOperand(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset())); - // temp = temp[index_in_cache]; - __ Ldr(temp, HeapOperand(temp, index_in_cache)); - // lr = temp->entry_point_from_quick_compiled_code_; - __ Ldr(lr, HeapOperand(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArm64WordSize))); - // lr(); - __ Blr(lr); + LoadCurrentMethod(temp); + if (!invoke->IsRecursive()) { + // temp = temp->dex_cache_resolved_methods_; + __ Ldr(temp, HeapOperand(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset())); + // temp = temp[index_in_cache]; + __ Ldr(temp, HeapOperand(temp, index_in_cache)); + // lr = temp->entry_point_from_quick_compiled_code_; + __ Ldr(lr, HeapOperand(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kArm64WordSize))); + // lr(); + __ Blr(lr); + } else { + __ Bl(&frame_entry_label_); + } - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); - DCHECK(!codegen_->IsLeafMethod()); + RecordPcInfo(invoke, invoke->GetDexPc()); + DCHECK(!IsLeafMethod()); +} + +void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { + if (TryGenerateIntrinsicCode(invoke, codegen_)) { + return; + } + + Register temp = WRegisterFrom(invoke->GetLocations()->GetTemp(0)); + codegen_->GenerateStaticOrDirectCall(invoke, temp); } void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { + if (TryGenerateIntrinsicCode(invoke, codegen_)) { + return; + } + LocationSummary* locations = invoke->GetLocations(); Location receiver = locations->InAt(0); Register temp = WRegisterFrom(invoke->GetLocations()->GetTemp(0)); @@ -2356,8 +2242,8 @@ void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction) { void LocationsBuilderARM64::VisitRem(HRem* rem) { Primitive::Type type = rem->GetResultType(); - LocationSummary::CallKind call_kind = IsFPType(type) ? LocationSummary::kCall - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = + Primitive::IsFloatingPointType(type) ? LocationSummary::kCall : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); switch (type) { @@ -2592,13 +2478,13 @@ void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) { LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type; } - if (IsFPType(input_type)) { + if (Primitive::IsFloatingPointType(input_type)) { locations->SetInAt(0, Location::RequiresFpuRegister()); } else { locations->SetInAt(0, Location::RequiresRegister()); } - if (IsFPType(result_type)) { + if (Primitive::IsFloatingPointType(result_type)) { locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); } else { locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); @@ -2611,7 +2497,7 @@ void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* convers DCHECK_NE(input_type, result_type); - if (IsIntegralType(result_type) && IsIntegralType(input_type)) { + if (Primitive::IsIntegralType(result_type) && Primitive::IsIntegralType(input_type)) { int result_size = Primitive::ComponentSize(result_type); int input_size = Primitive::ComponentSize(input_type); int min_size = std::min(result_size, input_size); @@ -2625,12 +2511,13 @@ void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* convers } else { __ Sbfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte); } - } else if (IsFPType(result_type) && IsIntegralType(input_type)) { + } else if (Primitive::IsFloatingPointType(result_type) && Primitive::IsIntegralType(input_type)) { __ Scvtf(OutputFPRegister(conversion), InputRegisterAt(conversion, 0)); - } else if (IsIntegralType(result_type) && IsFPType(input_type)) { + } else if (Primitive::IsIntegralType(result_type) && Primitive::IsFloatingPointType(input_type)) { CHECK(result_type == Primitive::kPrimInt || result_type == Primitive::kPrimLong); __ Fcvtzs(OutputRegister(conversion), InputFPRegisterAt(conversion, 0)); - } else if (IsFPType(result_type) && IsFPType(input_type)) { + } else if (Primitive::IsFloatingPointType(result_type) && + Primitive::IsFloatingPointType(input_type)) { __ Fcvt(OutputFPRegister(conversion), InputFPRegisterAt(conversion, 0)); } else { LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 96013e55c6..9a99dcccea 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -31,7 +31,10 @@ namespace art { namespace arm64 { class CodeGeneratorARM64; -class SlowPathCodeARM64; + +// TODO: Tune the use of Load-Acquire, Store-Release vs Data Memory Barriers. +// For now we prefer the use of load-acquire, store-release over explicit memory barriers. +static constexpr bool kUseAcquireRelease = true; // Use a local definition to prevent copying mistakes. static constexpr size_t kArm64WordSize = kArm64PointerSize; @@ -45,7 +48,8 @@ static const vixl::FPRegister kParameterFPRegisters[] = { }; static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters); -const vixl::Register tr = vixl::x18; // Thread Register +const vixl::Register tr = vixl::x18; // Thread Register +static const vixl::Register kArtMethodRegister = vixl::w0; // Method register on invoke. const vixl::CPURegList vixl_reserved_core_registers(vixl::ip0, vixl::ip1); const vixl::CPURegList vixl_reserved_fp_registers(vixl::d31); @@ -56,6 +60,20 @@ const vixl::CPURegList quick_callee_saved_registers(vixl::CPURegister::kRegister Location ARM64ReturnLocation(Primitive::Type return_type); +class SlowPathCodeARM64 : public SlowPathCode { + public: + SlowPathCodeARM64() : entry_label_(), exit_label_() {} + + vixl::Label* GetEntryLabel() { return &entry_label_; } + vixl::Label* GetExitLabel() { return &exit_label_; } + + private: + vixl::Label entry_label_; + vixl::Label exit_label_; + + DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64); +}; + class InvokeDexCallingConvention : public CallingConvention<vixl::Register, vixl::FPRegister> { public: InvokeDexCallingConvention() @@ -274,9 +292,12 @@ class CodeGeneratorARM64 : public CodeGenerator { return false; } + void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, vixl::Register temp); + private: // Labels for each block that will be compiled. vixl::Label* block_labels_; + vixl::Label frame_entry_label_; LocationsBuilderARM64 location_builder_; InstructionCodeGeneratorARM64 instruction_visitor_; diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 66f1d5e58d..063550be92 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -359,6 +359,16 @@ size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id return kX86WordSize; } +size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { + __ movsd(Address(ESP, stack_index), XmmRegister(reg_id)); + return GetFloatingPointSpillSlotSize(); +} + +size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { + __ movsd(XmmRegister(reg_id), Address(ESP, stack_index)); + return GetFloatingPointSpillSlotSize(); +} + CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, const CompilerOptions& compiler_options) : CodeGenerator(graph, kNumberOfCpuRegisters, kNumberOfXmmRegisters, kNumberOfRegisterPairs, (1 << kFakeReturnRegister), 0, compiler_options), @@ -449,6 +459,7 @@ InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGene codegen_(codegen) {} void CodeGeneratorX86::GenerateFrameEntry() { + __ Bind(&frame_entry_label_); bool skip_overflow_check = IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86); DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); @@ -586,6 +597,16 @@ void CodeGeneratorX86::Move32(Location destination, Location source) { __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>()); } else if (source.IsFpuRegister()) { __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>()); + } else if (source.IsConstant()) { + HConstant* constant = source.GetConstant(); + int32_t value; + if (constant->IsIntConstant()) { + value = constant->AsIntConstant()->GetValue(); + } else { + DCHECK(constant->IsFloatConstant()); + value = bit_cast<float, int32_t>(constant->AsFloatConstant()->GetValue()); + } + __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value)); } else { DCHECK(source.IsStackSlot()); __ pushl(Address(ESP, source.GetStackIndex())); @@ -642,7 +663,9 @@ void CodeGeneratorX86::Move64(Location destination, Location source) { __ movl(calling_convention.GetRegisterAt(register_index), Address(ESP, source.GetStackIndex())); } } else if (destination.IsFpuRegister()) { - if (source.IsDoubleStackSlot()) { + if (source.IsFpuRegister()) { + __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>()); + } else if (source.IsDoubleStackSlot()) { __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex())); } else { LOG(FATAL) << "Unimplemented"; @@ -1125,13 +1148,17 @@ void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirec // temp = method; codegen_->LoadCurrentMethod(temp); - // temp = temp->dex_cache_resolved_methods_; - __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value())); - // temp = temp[index_in_cache] - __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex()))); - // (temp + offset_of_quick_compiled_code)() - __ call(Address( - temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); + if (!invoke->IsRecursive()) { + // temp = temp->dex_cache_resolved_methods_; + __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().Int32Value())); + // temp = temp[index_in_cache] + __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex()))); + // (temp + offset_of_quick_compiled_code)() + __ call(Address( + temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); + } else { + __ call(codegen_->GetFrameEntryLabel()); + } DCHECK(!codegen_->IsLeafMethod()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); @@ -3113,10 +3140,30 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { break; } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - LOG(FATAL) << "Unimplemented register type " << type; - UNREACHABLE(); + case Primitive::kPrimFloat: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value(); + XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + if (index.IsConstant()) { + __ movss(out, Address(obj, + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset)); + } else { + __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset)); + } + break; + } + + case Primitive::kPrimDouble: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value(); + XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + if (index.IsConstant()) { + __ movsd(out, Address(obj, + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset)); + } else { + __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset)); + } + break; + } + case Primitive::kPrimVoid: LOG(FATAL) << "Unreachable type " << type; UNREACHABLE(); @@ -3305,10 +3352,32 @@ void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) { break; } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - LOG(FATAL) << "Unimplemented register type " << instruction->GetType(); - UNREACHABLE(); + case Primitive::kPrimFloat: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value(); + DCHECK(value.IsFpuRegister()); + if (index.IsConstant()) { + size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>()); + } else { + __ movss(Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset), + value.AsFpuRegister<XmmRegister>()); + } + break; + } + + case Primitive::kPrimDouble: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value(); + DCHECK(value.IsFpuRegister()); + if (index.IsConstant()) { + size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; + __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>()); + } else { + __ movsd(Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset), + value.AsFpuRegister<XmmRegister>()); + } + break; + } + case Primitive::kPrimVoid: LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); @@ -3410,12 +3479,24 @@ X86Assembler* ParallelMoveResolverX86::GetAssembler() const { return codegen_->GetAssembler(); } -void ParallelMoveResolverX86::MoveMemoryToMemory(int dst, int src) { +void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) { ScratchRegisterScope ensure_scratch( this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters()); + Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister()); int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0; - __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, src + stack_offset)); - __ movl(Address(ESP, dst + stack_offset), static_cast<Register>(ensure_scratch.GetRegister())); + __ movl(temp_reg, Address(ESP, src + stack_offset)); + __ movl(Address(ESP, dst + stack_offset), temp_reg); +} + +void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) { + ScratchRegisterScope ensure_scratch( + this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters()); + Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister()); + int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0; + __ movl(temp_reg, Address(ESP, src + stack_offset)); + __ movl(Address(ESP, dst + stack_offset), temp_reg); + __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize)); + __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg); } void ParallelMoveResolverX86::EmitMove(size_t index) { @@ -3430,21 +3511,55 @@ void ParallelMoveResolverX86::EmitMove(size_t index) { DCHECK(destination.IsStackSlot()); __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>()); } + } else if (source.IsFpuRegister()) { + if (destination.IsFpuRegister()) { + __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>()); + } else if (destination.IsStackSlot()) { + __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>()); + } else { + DCHECK(destination.IsDoubleStackSlot()); + __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>()); + } } else if (source.IsStackSlot()) { if (destination.IsRegister()) { __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex())); + } else if (destination.IsFpuRegister()) { + __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex())); } else { DCHECK(destination.IsStackSlot()); - MoveMemoryToMemory(destination.GetStackIndex(), - source.GetStackIndex()); + MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex()); + } + } else if (source.IsDoubleStackSlot()) { + if (destination.IsFpuRegister()) { + __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex())); + } else { + DCHECK(destination.IsDoubleStackSlot()) << destination; + MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex()); } } else if (source.IsConstant()) { - HIntConstant* instruction = source.GetConstant()->AsIntConstant(); - Immediate imm(instruction->AsIntConstant()->GetValue()); - if (destination.IsRegister()) { - __ movl(destination.AsRegister<Register>(), imm); + HConstant* constant = source.GetConstant(); + if (constant->IsIntConstant()) { + Immediate imm(constant->AsIntConstant()->GetValue()); + if (destination.IsRegister()) { + __ movl(destination.AsRegister<Register>(), imm); + } else { + DCHECK(destination.IsStackSlot()) << destination; + __ movl(Address(ESP, destination.GetStackIndex()), imm); + } } else { - __ movl(Address(ESP, destination.GetStackIndex()), imm); + DCHECK(constant->IsFloatConstant()); + float value = constant->AsFloatConstant()->GetValue(); + Immediate imm(bit_cast<float, int32_t>(value)); + if (destination.IsFpuRegister()) { + ScratchRegisterScope ensure_scratch( + this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters()); + Register temp = static_cast<Register>(ensure_scratch.GetRegister()); + __ movl(temp, imm); + __ movd(destination.AsFpuRegister<XmmRegister>(), temp); + } else { + DCHECK(destination.IsStackSlot()) << destination; + __ movl(Address(ESP, destination.GetStackIndex()), imm); + } } } else { LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source; @@ -3462,6 +3577,17 @@ void ParallelMoveResolverX86::Exchange(Register reg, int mem) { __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister())); } +void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) { + ScratchRegisterScope ensure_scratch( + this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters()); + + Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister()); + int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0; + __ movl(temp_reg, Address(ESP, mem + stack_offset)); + __ movss(Address(ESP, mem + stack_offset), reg); + __ movd(reg, temp_reg); +} + void ParallelMoveResolverX86::Exchange(int mem1, int mem2) { ScratchRegisterScope ensure_scratch1( this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters()); @@ -3491,8 +3617,18 @@ void ParallelMoveResolverX86::EmitSwap(size_t index) { Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); } else if (source.IsStackSlot() && destination.IsStackSlot()) { Exchange(destination.GetStackIndex(), source.GetStackIndex()); + } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { + // Use XOR Swap algorithm to avoid a temporary. + DCHECK_NE(source.reg(), destination.reg()); + __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>()); + __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>()); + __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>()); + } else if (source.IsFpuRegister() && destination.IsStackSlot()) { + Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex()); + } else if (destination.IsFpuRegister() && source.IsStackSlot()) { + Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex()); } else { - LOG(FATAL) << "Unimplemented"; + LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination; } } diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 55d71e39c4..107ddafea4 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -89,7 +89,9 @@ class ParallelMoveResolverX86 : public ParallelMoveResolver { private: void Exchange(Register reg, int mem); void Exchange(int mem1, int mem2); - void MoveMemoryToMemory(int dst, int src); + void Exchange32(XmmRegister reg, int mem); + void MoveMemoryToMemory32(int dst, int src); + void MoveMemoryToMemory64(int dst, int src); CodeGeneratorX86* const codegen_; @@ -173,6 +175,8 @@ class CodeGeneratorX86 : public CodeGenerator { void Move(HInstruction* instruction, Location location, HInstruction* move_for) OVERRIDE; size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; size_t GetWordSize() const OVERRIDE { return kX86WordSize; @@ -241,9 +245,12 @@ class CodeGeneratorX86 : public CodeGenerator { return type == Primitive::kPrimLong; } + Label* GetFrameEntryLabel() { return &frame_entry_label_; } + private: // Labels for each block that will be compiled. GrowableArray<Label> block_labels_; + Label frame_entry_label_; LocationsBuilderX86 location_builder_; InstructionCodeGeneratorX86 instruction_visitor_; ParallelMoveResolverX86 move_resolver_; diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 6bc28ff247..90b7bdaac3 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -361,13 +361,17 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo // temp = method; LoadCurrentMethod(temp); - // temp = temp->dex_cache_resolved_methods_; - __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue())); - // temp = temp[index_in_cache] - __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex()))); - // (temp + offset_of_quick_compiled_code)() - __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kX86_64WordSize).SizeValue())); + if (!invoke->IsRecursive()) { + // temp = temp->dex_cache_resolved_methods_; + __ movl(temp, Address(temp, mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue())); + // temp = temp[index_in_cache] + __ movl(temp, Address(temp, CodeGenerator::GetCacheOffset(invoke->GetDexMethodIndex()))); + // (temp + offset_of_quick_compiled_code)() + __ call(Address(temp, mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kX86_64WordSize).SizeValue())); + } else { + __ call(&frame_entry_label_); + } DCHECK(!IsLeafMethod()); RecordPcInfo(invoke, invoke->GetDexPc()); @@ -472,6 +476,7 @@ void CodeGeneratorX86_64::SetupBlockedRegisters(bool is_baseline) const { } void CodeGeneratorX86_64::GenerateFrameEntry() { + __ Bind(&frame_entry_label_); bool skip_overflow_check = IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86_64); DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 1ac2ab76a4..dbdbf869db 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -37,8 +37,6 @@ static constexpr FloatRegister kParameterFloatRegisters[] = static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); static constexpr size_t kParameterFloatRegistersLength = arraysize(kParameterFloatRegisters); -static constexpr bool kCoalescedImplicitNullCheck = false; - class InvokeDexCallingConvention : public CallingConvention<Register, FloatRegister> { public: InvokeDexCallingConvention() : CallingConvention( @@ -250,6 +248,7 @@ class CodeGeneratorX86_64 : public CodeGenerator { private: // Labels for each block that will be compiled. GrowableArray<Label> block_labels_; + Label frame_entry_label_; LocationsBuilderX86_64 location_builder_; InstructionCodeGeneratorX86_64 instruction_visitor_; ParallelMoveResolverX86_64 move_resolver_; diff --git a/compiler/optimizing/common_arm64.h b/compiler/optimizing/common_arm64.h new file mode 100644 index 0000000000..007324eb68 --- /dev/null +++ b/compiler/optimizing/common_arm64.h @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_COMMON_ARM64_H_ +#define ART_COMPILER_OPTIMIZING_COMMON_ARM64_H_ + +#include "locations.h" +#include "nodes.h" +#include "utils/arm64/assembler_arm64.h" +#include "a64/disasm-a64.h" +#include "a64/macro-assembler-a64.h" + +namespace art { +namespace arm64 { +namespace helpers { + +// Convenience helpers to ease conversion to and from VIXL operands. +static_assert((SP == 31) && (WSP == 31) && (XZR == 32) && (WZR == 32), + "Unexpected values for register codes."); + +static inline int VIXLRegCodeFromART(int code) { + if (code == SP) { + return vixl::kSPRegInternalCode; + } + if (code == XZR) { + return vixl::kZeroRegCode; + } + return code; +} + +static inline int ARTRegCodeFromVIXL(int code) { + if (code == vixl::kSPRegInternalCode) { + return SP; + } + if (code == vixl::kZeroRegCode) { + return XZR; + } + return code; +} + +static inline vixl::Register XRegisterFrom(Location location) { + DCHECK(location.IsRegister()); + return vixl::Register::XRegFromCode(VIXLRegCodeFromART(location.reg())); +} + +static inline vixl::Register WRegisterFrom(Location location) { + DCHECK(location.IsRegister()); + return vixl::Register::WRegFromCode(VIXLRegCodeFromART(location.reg())); +} + +static inline vixl::Register RegisterFrom(Location location, Primitive::Type type) { + DCHECK(type != Primitive::kPrimVoid && !Primitive::IsFloatingPointType(type)); + return type == Primitive::kPrimLong ? XRegisterFrom(location) : WRegisterFrom(location); +} + +static inline vixl::Register OutputRegister(HInstruction* instr) { + return RegisterFrom(instr->GetLocations()->Out(), instr->GetType()); +} + +static inline vixl::Register InputRegisterAt(HInstruction* instr, int input_index) { + return RegisterFrom(instr->GetLocations()->InAt(input_index), + instr->InputAt(input_index)->GetType()); +} + +static inline vixl::FPRegister DRegisterFrom(Location location) { + DCHECK(location.IsFpuRegister()); + return vixl::FPRegister::DRegFromCode(location.reg()); +} + +static inline vixl::FPRegister SRegisterFrom(Location location) { + DCHECK(location.IsFpuRegister()); + return vixl::FPRegister::SRegFromCode(location.reg()); +} + +static inline vixl::FPRegister FPRegisterFrom(Location location, Primitive::Type type) { + DCHECK(Primitive::IsFloatingPointType(type)); + return type == Primitive::kPrimDouble ? DRegisterFrom(location) : SRegisterFrom(location); +} + +static inline vixl::FPRegister OutputFPRegister(HInstruction* instr) { + return FPRegisterFrom(instr->GetLocations()->Out(), instr->GetType()); +} + +static inline vixl::FPRegister InputFPRegisterAt(HInstruction* instr, int input_index) { + return FPRegisterFrom(instr->GetLocations()->InAt(input_index), + instr->InputAt(input_index)->GetType()); +} + +static inline vixl::CPURegister CPURegisterFrom(Location location, Primitive::Type type) { + return Primitive::IsFloatingPointType(type) ? vixl::CPURegister(FPRegisterFrom(location, type)) + : vixl::CPURegister(RegisterFrom(location, type)); +} + +static inline vixl::CPURegister OutputCPURegister(HInstruction* instr) { + return Primitive::IsFloatingPointType(instr->GetType()) + ? static_cast<vixl::CPURegister>(OutputFPRegister(instr)) + : static_cast<vixl::CPURegister>(OutputRegister(instr)); +} + +static inline vixl::CPURegister InputCPURegisterAt(HInstruction* instr, int index) { + return Primitive::IsFloatingPointType(instr->InputAt(index)->GetType()) + ? static_cast<vixl::CPURegister>(InputFPRegisterAt(instr, index)) + : static_cast<vixl::CPURegister>(InputRegisterAt(instr, index)); +} + +static inline int64_t Int64ConstantFrom(Location location) { + HConstant* instr = location.GetConstant(); + return instr->IsIntConstant() ? instr->AsIntConstant()->GetValue() + : instr->AsLongConstant()->GetValue(); +} + +static inline vixl::Operand OperandFrom(Location location, Primitive::Type type) { + if (location.IsRegister()) { + return vixl::Operand(RegisterFrom(location, type)); + } else { + return vixl::Operand(Int64ConstantFrom(location)); + } +} + +static inline vixl::Operand InputOperandAt(HInstruction* instr, int input_index) { + return OperandFrom(instr->GetLocations()->InAt(input_index), + instr->InputAt(input_index)->GetType()); +} + +static inline vixl::MemOperand StackOperandFrom(Location location) { + return vixl::MemOperand(vixl::sp, location.GetStackIndex()); +} + +static inline vixl::MemOperand HeapOperand(const vixl::Register& base, size_t offset = 0) { + // A heap reference must be 32bit, so fit in a W register. + DCHECK(base.IsW()); + return vixl::MemOperand(base.X(), offset); +} + +static inline vixl::MemOperand HeapOperand(const vixl::Register& base, Offset offset) { + return HeapOperand(base, offset.SizeValue()); +} + +static inline vixl::MemOperand HeapOperandFrom(Location location, Offset offset) { + return HeapOperand(RegisterFrom(location, Primitive::kPrimNot), offset); +} + +static inline Location LocationFrom(const vixl::Register& reg) { + return Location::RegisterLocation(ARTRegCodeFromVIXL(reg.code())); +} + +static inline Location LocationFrom(const vixl::FPRegister& fpreg) { + return Location::FpuRegisterLocation(fpreg.code()); +} + +static inline vixl::Operand OperandFromMemOperand(const vixl::MemOperand& mem_op) { + if (mem_op.IsImmediateOffset()) { + return vixl::Operand(mem_op.offset()); + } else { + DCHECK(mem_op.IsRegisterOffset()); + if (mem_op.extend() != vixl::NO_EXTEND) { + return vixl::Operand(mem_op.regoffset(), mem_op.extend(), mem_op.shift_amount()); + } else if (mem_op.shift() != vixl::NO_SHIFT) { + return vixl::Operand(mem_op.regoffset(), mem_op.shift(), mem_op.shift_amount()); + } else { + LOG(FATAL) << "Should not reach here"; + UNREACHABLE(); + } + } +} + +} // namespace helpers +} // namespace arm64 +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_COMMON_ARM64_H_ diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index 63bc4ae671..17c8f337ca 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -27,6 +27,7 @@ class InstructionSimplifierVisitor : public HGraphVisitor { void VisitEqual(HEqual* equal) OVERRIDE; void VisitArraySet(HArraySet* equal) OVERRIDE; void VisitTypeConversion(HTypeConversion* instruction) OVERRIDE; + void VisitNullCheck(HNullCheck* instruction) OVERRIDE; }; void InstructionSimplifier::Run() { @@ -34,6 +35,14 @@ void InstructionSimplifier::Run() { visitor.VisitInsertionOrder(); } +void InstructionSimplifierVisitor::VisitNullCheck(HNullCheck* null_check) { + HInstruction* obj = null_check->InputAt(0); + if (!obj->CanBeNull()) { + null_check->ReplaceWith(obj); + null_check->GetBlock()->RemoveInstruction(null_check); + } +} + void InstructionSimplifierVisitor::VisitSuspendCheck(HSuspendCheck* check) { HBasicBlock* block = check->GetBlock(); // Currently always keep the suspend check at entry. diff --git a/compiler/optimizing/instruction_simplifier.h b/compiler/optimizing/instruction_simplifier.h index 7068c7fc10..bca6697d05 100644 --- a/compiler/optimizing/instruction_simplifier.h +++ b/compiler/optimizing/instruction_simplifier.h @@ -27,8 +27,8 @@ namespace art { */ class InstructionSimplifier : public HOptimization { public: - explicit InstructionSimplifier(HGraph* graph) - : HOptimization(graph, true, "instruction_simplifier") {} + explicit InstructionSimplifier(HGraph* graph, const char* name = "instruction_simplifier") + : HOptimization(graph, true, name) {} void Run() OVERRIDE; }; diff --git a/compiler/optimizing/intrinsics.cc b/compiler/optimizing/intrinsics.cc index fe0e7f2eb2..36cf8568e5 100644 --- a/compiler/optimizing/intrinsics.cc +++ b/compiler/optimizing/intrinsics.cc @@ -47,25 +47,25 @@ static Primitive::Type GetType(uint64_t data, bool is_op_size) { if (is_op_size) { switch (static_cast<OpSize>(data)) { case kSignedByte: - return Primitive::Type::kPrimByte; + return Primitive::kPrimByte; case kSignedHalf: - return Primitive::Type::kPrimShort; + return Primitive::kPrimShort; case k32: - return Primitive::Type::kPrimInt; + return Primitive::kPrimInt; case k64: - return Primitive::Type::kPrimLong; + return Primitive::kPrimLong; default: LOG(FATAL) << "Unknown/unsupported op size " << data; UNREACHABLE(); } } else { if ((data & kIntrinsicFlagIsLong) != 0) { - return Primitive::Type::kPrimLong; + return Primitive::kPrimLong; } if ((data & kIntrinsicFlagIsObject) != 0) { - return Primitive::Type::kPrimNot; + return Primitive::kPrimNot; } - return Primitive::Type::kPrimInt; + return Primitive::kPrimInt; } } @@ -82,9 +82,9 @@ static Intrinsics GetIntrinsic(InlineMethod method) { // Bit manipulations. case kIntrinsicReverseBits: switch (GetType(method.d.data, true)) { - case Primitive::Type::kPrimInt: + case Primitive::kPrimInt: return Intrinsics::kIntegerReverse; - case Primitive::Type::kPrimLong: + case Primitive::kPrimLong: return Intrinsics::kLongReverse; default: LOG(FATAL) << "Unknown/unsupported op size " << method.d.data; @@ -93,11 +93,11 @@ static Intrinsics GetIntrinsic(InlineMethod method) { break; case kIntrinsicReverseBytes: switch (GetType(method.d.data, true)) { - case Primitive::Type::kPrimShort: + case Primitive::kPrimShort: return Intrinsics::kShortReverseBytes; - case Primitive::Type::kPrimInt: + case Primitive::kPrimInt: return Intrinsics::kIntegerReverseBytes; - case Primitive::Type::kPrimLong: + case Primitive::kPrimLong: return Intrinsics::kLongReverseBytes; default: LOG(FATAL) << "Unknown/unsupported op size " << method.d.data; @@ -154,13 +154,13 @@ static Intrinsics GetIntrinsic(InlineMethod method) { // Memory.peek. case kIntrinsicPeek: switch (GetType(method.d.data, true)) { - case Primitive::Type::kPrimByte: + case Primitive::kPrimByte: return Intrinsics::kMemoryPeekByte; - case Primitive::Type::kPrimShort: + case Primitive::kPrimShort: return Intrinsics::kMemoryPeekShortNative; - case Primitive::Type::kPrimInt: + case Primitive::kPrimInt: return Intrinsics::kMemoryPeekIntNative; - case Primitive::Type::kPrimLong: + case Primitive::kPrimLong: return Intrinsics::kMemoryPeekLongNative; default: LOG(FATAL) << "Unknown/unsupported op size " << method.d.data; @@ -171,13 +171,13 @@ static Intrinsics GetIntrinsic(InlineMethod method) { // Memory.poke. case kIntrinsicPoke: switch (GetType(method.d.data, true)) { - case Primitive::Type::kPrimByte: + case Primitive::kPrimByte: return Intrinsics::kMemoryPokeByte; - case Primitive::Type::kPrimShort: + case Primitive::kPrimShort: return Intrinsics::kMemoryPokeShortNative; - case Primitive::Type::kPrimInt: + case Primitive::kPrimInt: return Intrinsics::kMemoryPokeIntNative; - case Primitive::Type::kPrimLong: + case Primitive::kPrimLong: return Intrinsics::kMemoryPokeLongNative; default: LOG(FATAL) << "Unknown/unsupported op size " << method.d.data; @@ -199,11 +199,11 @@ static Intrinsics GetIntrinsic(InlineMethod method) { case kIntrinsicCas: switch (GetType(method.d.data, false)) { - case Primitive::Type::kPrimNot: + case Primitive::kPrimNot: return Intrinsics::kUnsafeCASObject; - case Primitive::Type::kPrimInt: + case Primitive::kPrimInt: return Intrinsics::kUnsafeCASInt; - case Primitive::Type::kPrimLong: + case Primitive::kPrimLong: return Intrinsics::kUnsafeCASLong; default: LOG(FATAL) << "Unknown/unsupported op size " << method.d.data; @@ -213,10 +213,12 @@ static Intrinsics GetIntrinsic(InlineMethod method) { case kIntrinsicUnsafeGet: { const bool is_volatile = (method.d.data & kIntrinsicFlagIsVolatile); switch (GetType(method.d.data, false)) { - case Primitive::Type::kPrimInt: + case Primitive::kPrimInt: return is_volatile ? Intrinsics::kUnsafeGetVolatile : Intrinsics::kUnsafeGet; - case Primitive::Type::kPrimLong: + case Primitive::kPrimLong: return is_volatile ? Intrinsics::kUnsafeGetLongVolatile : Intrinsics::kUnsafeGetLong; + case Primitive::kPrimNot: + return is_volatile ? Intrinsics::kUnsafeGetObjectVolatile : Intrinsics::kUnsafeGetObject; default: LOG(FATAL) << "Unknown/unsupported op size " << method.d.data; UNREACHABLE(); @@ -230,7 +232,7 @@ static Intrinsics GetIntrinsic(InlineMethod method) { ((method.d.data & kIntrinsicFlagIsOrdered) != 0) ? kOrdered : kNoSync; switch (GetType(method.d.data, false)) { - case Primitive::Type::kPrimInt: + case Primitive::kPrimInt: switch (sync) { case kNoSync: return Intrinsics::kUnsafePut; @@ -240,7 +242,7 @@ static Intrinsics GetIntrinsic(InlineMethod method) { return Intrinsics::kUnsafePutOrdered; } break; - case Primitive::Type::kPrimLong: + case Primitive::kPrimLong: switch (sync) { case kNoSync: return Intrinsics::kUnsafePutLong; @@ -250,7 +252,7 @@ static Intrinsics GetIntrinsic(InlineMethod method) { return Intrinsics::kUnsafePutLongOrdered; } break; - case Primitive::Type::kPrimNot: + case Primitive::kPrimNot: switch (sync) { case kNoSync: return Intrinsics::kUnsafePutObject; diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc new file mode 100644 index 0000000000..ba26afed44 --- /dev/null +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -0,0 +1,1000 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "intrinsics_arm64.h" + +#include "code_generator_arm64.h" +#include "common_arm64.h" +#include "entrypoints/quick/quick_entrypoints.h" +#include "intrinsics.h" +#include "mirror/array-inl.h" +#include "mirror/art_method.h" +#include "mirror/string.h" +#include "thread.h" +#include "utils/arm64/assembler_arm64.h" +#include "utils/arm64/constants_arm64.h" + +#include "a64/disasm-a64.h" +#include "a64/macro-assembler-a64.h" + +using namespace vixl; // NOLINT(build/namespaces) + +namespace art { + +namespace arm64 { + +using helpers::DRegisterFrom; +using helpers::FPRegisterFrom; +using helpers::HeapOperand; +using helpers::RegisterFrom; +using helpers::SRegisterFrom; +using helpers::WRegisterFrom; +using helpers::XRegisterFrom; + + +namespace { + +ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) { + return MemOperand(XRegisterFrom(location), offset); +} + +} // namespace + +vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() { + return codegen_->GetAssembler()->vixl_masm_; +} + +ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() { + return codegen_->GetGraph()->GetArena(); +} + +#define __ codegen->GetAssembler()->vixl_masm_-> + +static void MoveFromReturnRegister(Location trg, + Primitive::Type type, + CodeGeneratorARM64* codegen) { + if (!trg.IsValid()) { + DCHECK(type == Primitive::kPrimVoid); + return; + } + + DCHECK_NE(type, Primitive::kPrimVoid); + + if (Primitive::IsIntegralType(type)) { + Register trg_reg = RegisterFrom(trg, type); + Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type); + __ Mov(trg_reg, res_reg, kDiscardForSameWReg); + } else { + FPRegister trg_reg = FPRegisterFrom(trg, type); + FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type); + __ Fmov(trg_reg, res_reg); + } +} + +static void MoveArguments(HInvoke* invoke, ArenaAllocator* arena, CodeGeneratorARM64* codegen) { + if (invoke->InputCount() == 0) { + return; + } + + LocationSummary* locations = invoke->GetLocations(); + InvokeDexCallingConventionVisitor calling_convention_visitor; + + // We're moving potentially two or more locations to locations that could overlap, so we need + // a parallel move resolver. + HParallelMove parallel_move(arena); + + for (size_t i = 0; i < invoke->InputCount(); i++) { + HInstruction* input = invoke->InputAt(i); + Location cc_loc = calling_convention_visitor.GetNextLocation(input->GetType()); + Location actual_loc = locations->InAt(i); + + parallel_move.AddMove(actual_loc, cc_loc, nullptr); + } + + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); +} + +// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified +// call. This will copy the arguments into the positions for a regular call. +// +// Note: The actual parameters are required to be in the locations given by the invoke's location +// summary. If an intrinsic modifies those locations before a slowpath call, they must be +// restored! +class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 { + public: + explicit IntrinsicSlowPathARM64(HInvoke* invoke) : invoke_(invoke) { } + + void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE { + CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in); + __ Bind(GetEntryLabel()); + + codegen->SaveLiveRegisters(invoke_->GetLocations()); + + MoveArguments(invoke_, codegen->GetGraph()->GetArena(), codegen); + + if (invoke_->IsInvokeStaticOrDirect()) { + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), kArtMethodRegister); + } else { + UNIMPLEMENTED(FATAL) << "Non-direct intrinsic slow-path not yet implemented"; + UNREACHABLE(); + } + + // Copy the result back to the expected output. + Location out = invoke_->GetLocations()->Out(); + if (out.IsValid()) { + DCHECK(out.IsRegister()); // TODO: Replace this when we support output in memory. + DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg())); + MoveFromReturnRegister(out, invoke_->GetType(), codegen); + } + + codegen->RestoreLiveRegisters(invoke_->GetLocations()); + __ B(GetExitLabel()); + } + + private: + // The instruction where this slow path is happening. + HInvoke* const invoke_; + + DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64); +}; + +#undef __ + +bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) { + Dispatch(invoke); + LocationSummary* res = invoke->GetLocations(); + return res != nullptr && res->Intrinsified(); +} + +#define __ masm-> + +static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresRegister()); +} + +static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); +} + +static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { + Location input = locations->InAt(0); + Location output = locations->Out(); + __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output), + is64bit ? DRegisterFrom(input) : SRegisterFrom(input)); +} + +static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { + Location input = locations->InAt(0); + Location output = locations->Out(); + __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output), + is64bit ? XRegisterFrom(input) : WRegisterFrom(input)); +} + +void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { + CreateIntToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { + MoveFPToInt(invoke->GetLocations(), true, GetVIXLAssembler()); +} +void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { + MoveIntToFP(invoke->GetLocations(), true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { + CreateIntToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { + MoveFPToInt(invoke->GetLocations(), false, GetVIXLAssembler()); +} +void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { + MoveIntToFP(invoke->GetLocations(), false, GetVIXLAssembler()); +} + +static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +static void GenReverseBytes(LocationSummary* locations, + Primitive::Type type, + vixl::MacroAssembler* masm) { + Location in = locations->InAt(0); + Location out = locations->Out(); + + switch (type) { + case Primitive::kPrimShort: + __ Rev16(WRegisterFrom(out), WRegisterFrom(in)); + __ Sxth(WRegisterFrom(out), WRegisterFrom(out)); + break; + case Primitive::kPrimInt: + case Primitive::kPrimLong: + __ Rev(RegisterFrom(out, type), RegisterFrom(in, type)); + break; + default: + LOG(FATAL) << "Unexpected size for reverse-bytes: " << type; + UNREACHABLE(); + } +} + +void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) { + GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) { + GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) { + GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler()); +} + +static void GenReverse(LocationSummary* locations, + Primitive::Type type, + vixl::MacroAssembler* masm) { + DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); + + Location in = locations->InAt(0); + Location out = locations->Out(); + + __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); +} + +void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) { + GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) { + GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); +} + +static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { + // We only support FP registers here. + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); +} + +static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { + Location in = locations->InAt(0); + Location out = locations->Out(); + + FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in); + FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out); + + __ Fabs(out_reg, in_reg); +} + +void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) { + MathAbsFP(invoke->GetLocations(), true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) { + MathAbsFP(invoke->GetLocations(), false, GetVIXLAssembler()); +} + +static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +static void GenAbsInteger(LocationSummary* locations, + bool is64bit, + vixl::MacroAssembler* masm) { + Location in = locations->InAt(0); + Location output = locations->Out(); + + Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in); + Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output); + + __ Cmp(in_reg, Operand(0)); + __ Cneg(out_reg, in_reg, lt); +} + +void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) { + CreateIntToInt(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) { + GenAbsInteger(invoke->GetLocations(), false, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) { + CreateIntToInt(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) { + GenAbsInteger(invoke->GetLocations(), true, GetVIXLAssembler()); +} + +static void GenMinMaxFP(LocationSummary* locations, + bool is_min, + bool is_double, + vixl::MacroAssembler* masm) { + Location op1 = locations->InAt(0); + Location op2 = locations->InAt(1); + Location out = locations->Out(); + + FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1); + FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2); + FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out); + if (is_min) { + __ Fmin(out_reg, op1_reg, op2_reg); + } else { + __ Fmax(out_reg, op1_reg, op2_reg); + } +} + +static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { + GenMinMaxFP(invoke->GetLocations(), true, true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) { + GenMinMaxFP(invoke->GetLocations(), true, false, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { + GenMinMaxFP(invoke->GetLocations(), false, true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { + GenMinMaxFP(invoke->GetLocations(), false, false, GetVIXLAssembler()); +} + +static void GenMinMax(LocationSummary* locations, + bool is_min, + bool is_long, + vixl::MacroAssembler* masm) { + Location op1 = locations->InAt(0); + Location op2 = locations->InAt(1); + Location out = locations->Out(); + + Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1); + Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2); + Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out); + + __ Cmp(op1_reg, op2_reg); + __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt); +} + +static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) { + CreateIntIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) { + GenMinMax(invoke->GetLocations(), true, false, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) { + CreateIntIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) { + GenMinMax(invoke->GetLocations(), true, true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) { + CreateIntIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) { + GenMinMax(invoke->GetLocations(), false, false, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) { + CreateIntIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) { + GenMinMax(invoke->GetLocations(), false, true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) { + LocationSummary* locations = invoke->GetLocations(); + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); +} + +void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) { + LocationSummary* locations = invoke->GetLocations(); + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); +} + +void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) { + LocationSummary* locations = invoke->GetLocations(); + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); +} + +void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) { + LocationSummary* locations = invoke->GetLocations(); + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); +} + +static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresRegister()); +} + +static void GenMathRound(LocationSummary* locations, + bool is_double, + vixl::MacroAssembler* masm) { + FPRegister in_reg = is_double ? + DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0)); + Register out_reg = is_double ? + XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out()); + UseScratchRegisterScope temps(masm); + FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg); + + // 0.5 can be encoded as an immediate, so use fmov. + if (is_double) { + __ Fmov(temp1_reg, static_cast<double>(0.5)); + } else { + __ Fmov(temp1_reg, static_cast<float>(0.5)); + } + __ Fadd(temp1_reg, in_reg, temp1_reg); + __ Fcvtms(out_reg, temp1_reg); +} + +void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) { + CreateFPToIntPlusTempLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) { + GenMathRound(invoke->GetLocations(), true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) { + CreateFPToIntPlusTempLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) { + GenMathRound(invoke->GetLocations(), false, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)), + AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); +} + +void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetOut(Location::RequiresRegister()); +} + +void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) { + codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()), + MemOperand(tr, Thread::PeerOffset<8>().Int32Value())); +} + +static void GenUnsafeGet(HInvoke* invoke, + Primitive::Type type, + bool is_volatile, + CodeGeneratorARM64* codegen) { + LocationSummary* locations = invoke->GetLocations(); + DCHECK((type == Primitive::kPrimInt) || + (type == Primitive::kPrimLong) || + (type == Primitive::kPrimNot)); + vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; + Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. + Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. + Register trg = RegisterFrom(locations->Out(), type); + + MemOperand mem_op(base.X(), offset); + if (is_volatile) { + if (kUseAcquireRelease) { + codegen->LoadAcquire(invoke, trg, mem_op); + } else { + codegen->Load(type, trg, mem_op); + __ Dmb(InnerShareable, BarrierReads); + } + } else { + codegen->Load(type, trg, mem_op); + } +} + +static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimInt, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimInt, true, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimLong, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimLong, true, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimNot, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimNot, true, codegen_); +} + +static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetInAt(3, Location::RequiresRegister()); +} + +void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, invoke); +} + +static void GenUnsafePut(LocationSummary* locations, + Primitive::Type type, + bool is_volatile, + bool is_ordered, + CodeGeneratorARM64* codegen) { + vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; + + Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. + Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. + Register value = RegisterFrom(locations->InAt(3), type); + + MemOperand mem_op(base.X(), offset); + + if (is_volatile || is_ordered) { + if (kUseAcquireRelease) { + codegen->StoreRelease(type, value, mem_op); + } else { + __ Dmb(InnerShareable, BarrierAll); + codegen->Store(type, value, mem_op); + if (is_volatile) { + __ Dmb(InnerShareable, BarrierReads); + } + } + } else { + codegen->Store(type, value, mem_op); + } + + if (type == Primitive::kPrimNot) { + codegen->MarkGCCard(base, value); + } +} + +void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, true, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, true, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, true, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, true, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, false, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, true, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, true, false, codegen_); +} + +static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetInAt(3, Location::RequiresRegister()); + locations->SetInAt(4, Location::RequiresRegister()); + + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) { + // TODO: Currently we use acquire-release load-stores in the CAS loop. One could reasonably write + // a version relying on simple exclusive load-stores and barriers instead. + static_assert(kUseAcquireRelease, "Non-acquire-release inlined CAS not implemented, yet."); + + vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; + + Register out = WRegisterFrom(locations->Out()); // Boolean result. + + Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. + Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. + Register expected = RegisterFrom(locations->InAt(3), type); // Expected. + Register value = RegisterFrom(locations->InAt(4), type); // Value. + + // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps. + if (type == Primitive::kPrimNot) { + // Mark card for object assuming new value is stored. + codegen->MarkGCCard(base, value); + } + + UseScratchRegisterScope temps(masm); + Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory. + Register tmp_value = temps.AcquireSameSizeAs(value); // Value in memory. + + Register tmp_32 = tmp_value.W(); + + __ Add(tmp_ptr, base.X(), Operand(offset)); + + // do { + // tmp_value = [tmp_ptr] - expected; + // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value)); + // result = tmp_value != 0; + + vixl::Label loop_head, exit_loop; + __ Bind(&loop_head); + + __ Ldaxr(tmp_value, MemOperand(tmp_ptr)); + __ Cmp(tmp_value, expected); + __ B(&exit_loop, ne); + + __ Stlxr(tmp_32, value, MemOperand(tmp_ptr)); + __ Cbnz(tmp_32, &loop_head); + + __ Bind(&exit_loop); + __ Cset(out, eq); +} + +void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) { + CreateIntIntIntIntIntToInt(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) { + CreateIntIntIntIntIntToInt(arena_, invoke); +} +void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) { + CreateIntIntIntIntIntToInt(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) { + GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) { + GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_); +} +void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) { + GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_); +} + +void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) { + // The inputs plus one temp. + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kCallOnSlowPath, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + // Location of reference to data array + const MemberOffset value_offset = mirror::String::ValueOffset(); + // Location of count + const MemberOffset count_offset = mirror::String::CountOffset(); + // Starting offset within data array + const MemberOffset offset_offset = mirror::String::OffsetOffset(); + // Start of char data with array_ + const MemberOffset data_offset = mirror::Array::DataOffset(sizeof(uint16_t)); + + Register obj = WRegisterFrom(locations->InAt(0)); // String object pointer. + Register idx = WRegisterFrom(locations->InAt(1)); // Index of character. + Register out = WRegisterFrom(locations->Out()); // Result character. + + UseScratchRegisterScope temps(masm); + Register temp = temps.AcquireW(); + Register array_temp = temps.AcquireW(); // We can trade this for worse scheduling. + + // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth + // the cost. + // TODO: For simplicity, the index parameter is requested in a register, so different from Quick + // we will not optimize the code for constants (which would save a register). + + SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); + codegen_->AddSlowPath(slow_path); + + __ Ldr(temp, HeapOperand(obj, count_offset)); // temp = str.length. + codegen_->MaybeRecordImplicitNullCheck(invoke); + __ Cmp(idx, temp); + __ B(hs, slow_path->GetEntryLabel()); + + // Index computation. + __ Ldr(temp, HeapOperand(obj, offset_offset)); // temp := str.offset. + __ Ldr(array_temp, HeapOperand(obj, value_offset)); // array_temp := str.offset. + __ Add(temp, temp, idx); + DCHECK_EQ(data_offset.Int32Value() % 2, 0); // We'll compensate by shifting. + __ Add(temp, temp, Operand(data_offset.Int32Value() / 2)); + + // Load the value. + __ Ldrh(out, MemOperand(array_temp.X(), temp, UXTW, 1)); // out := array_temp[temp]. + + __ Bind(slow_path->GetExitLabel()); +} + +// Unimplemented intrinsics. + +#define UNIMPLEMENTED_INTRINSIC(Name) \ +void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ +} \ +void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ +} + +UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar) +UNIMPLEMENTED_INTRINSIC(StringCompareTo) +UNIMPLEMENTED_INTRINSIC(StringIsEmpty) // Might not want to do these two anyways, inlining should +UNIMPLEMENTED_INTRINSIC(StringLength) // be good enough here. +UNIMPLEMENTED_INTRINSIC(StringIndexOf) +UNIMPLEMENTED_INTRINSIC(StringIndexOfAfter) +UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) + +} // namespace arm64 +} // namespace art diff --git a/compiler/optimizing/intrinsics_arm64.h b/compiler/optimizing/intrinsics_arm64.h new file mode 100644 index 0000000000..ba21889839 --- /dev/null +++ b/compiler/optimizing/intrinsics_arm64.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_INTRINSICS_ARM64_H_ +#define ART_COMPILER_OPTIMIZING_INTRINSICS_ARM64_H_ + +#include "intrinsics.h" + +namespace vixl { + +class MacroAssembler; + +} // namespace vixl + +namespace art { + +class ArenaAllocator; +class HInvokeStaticOrDirect; +class HInvokeVirtual; + +namespace arm64 { + +class CodeGeneratorARM64; + +class IntrinsicLocationsBuilderARM64 FINAL : public IntrinsicVisitor { + public: + explicit IntrinsicLocationsBuilderARM64(ArenaAllocator* arena) : arena_(arena) {} + + // Define visitor methods. + +#define OPTIMIZING_INTRINSICS(Name, IsStatic) \ + void Visit ## Name(HInvoke* invoke) OVERRIDE; +#include "intrinsics_list.h" +INTRINSICS_LIST(OPTIMIZING_INTRINSICS) +#undef INTRINSICS_LIST +#undef OPTIMIZING_INTRINSICS + + // Check whether an invoke is an intrinsic, and if so, create a location summary. Returns whether + // a corresponding LocationSummary with the intrinsified_ flag set was generated and attached to + // the invoke. + bool TryDispatch(HInvoke* invoke); + + private: + ArenaAllocator* arena_; + + DISALLOW_COPY_AND_ASSIGN(IntrinsicLocationsBuilderARM64); +}; + +class IntrinsicCodeGeneratorARM64 FINAL : public IntrinsicVisitor { + public: + explicit IntrinsicCodeGeneratorARM64(CodeGeneratorARM64* codegen) : codegen_(codegen) {} + + // Define visitor methods. + +#define OPTIMIZING_INTRINSICS(Name, IsStatic) \ + void Visit ## Name(HInvoke* invoke) OVERRIDE; +#include "intrinsics_list.h" +INTRINSICS_LIST(OPTIMIZING_INTRINSICS) +#undef INTRINSICS_LIST +#undef OPTIMIZING_INTRINSICS + + private: + vixl::MacroAssembler* GetVIXLAssembler(); + + ArenaAllocator* GetAllocator(); + + CodeGeneratorARM64* codegen_; + + DISALLOW_COPY_AND_ASSIGN(IntrinsicCodeGeneratorARM64); +}; + +} // namespace arm64 +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_INTRINSICS_ARM64_H_ diff --git a/compiler/optimizing/intrinsics_list.h b/compiler/optimizing/intrinsics_list.h index 29ca20cca0..9cc77c6251 100644 --- a/compiler/optimizing/intrinsics_list.h +++ b/compiler/optimizing/intrinsics_list.h @@ -69,6 +69,8 @@ V(UnsafeCASObject, kDirect) \ V(UnsafeGet, kDirect) \ V(UnsafeGetVolatile, kDirect) \ + V(UnsafeGetObject, kDirect) \ + V(UnsafeGetObjectVolatile, kDirect) \ V(UnsafeGetLong, kDirect) \ V(UnsafeGetLongVolatile, kDirect) \ V(UnsafePut, kDirect) \ @@ -80,8 +82,7 @@ V(UnsafePutLong, kDirect) \ V(UnsafePutLongOrdered, kDirect) \ V(UnsafePutLongVolatile, kDirect) \ - \ - V(ReferenceGetReferent, kVirtual) + V(ReferenceGetReferent, kDirect) #endif // ART_COMPILER_OPTIMIZING_INTRINSICS_LIST_H_ #undef ART_COMPILER_OPTIMIZING_INTRINSICS_LIST_H_ // #define is only for lint. diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index 2c239458f1..c73f092a61 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -30,13 +30,11 @@ namespace art { namespace x86_64 { -static constexpr bool kIntrinsified = true; - X86_64Assembler* IntrinsicCodeGeneratorX86_64::GetAssembler() { return reinterpret_cast<X86_64Assembler*>(codegen_->GetAssembler()); } -ArenaAllocator* IntrinsicCodeGeneratorX86_64::GetArena() { +ArenaAllocator* IntrinsicCodeGeneratorX86_64::GetAllocator() { return codegen_->GetGraph()->GetArena(); } @@ -644,21 +642,18 @@ void IntrinsicCodeGeneratorX86_64::VisitStringCharAt(HInvoke* invoke) { Location temp_loc = locations->GetTemp(0); CpuRegister temp = temp_loc.AsRegister<CpuRegister>(); - // Note: Nullcheck has been done before in a HNullCheck before the HInvokeVirtual. If/when we - // move to (coalesced) implicit checks, we have to do a null check below. - DCHECK(!kCoalescedImplicitNullCheck); - // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth // the cost. // TODO: For simplicity, the index parameter is requested in a register, so different from Quick // we will not optimize the code for constants (which would save a register). - SlowPathCodeX86_64* slow_path = new (GetArena()) IntrinsicSlowPathX86_64(invoke); + SlowPathCodeX86_64* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); codegen_->AddSlowPath(slow_path); X86_64Assembler* assembler = GetAssembler(); __ cmpl(idx, Address(obj, count_offset)); + codegen_->MaybeRecordImplicitNullCheck(invoke); __ j(kAboveEqual, slow_path->GetEntryLabel()); // Get the actual element. @@ -803,18 +798,25 @@ void IntrinsicCodeGeneratorX86_64::VisitThreadCurrentThread(HInvoke* invoke) { GetAssembler()->gs()->movl(out, Address::Absolute(Thread::PeerOffset<kX86_64WordSize>(), true)); } -static void GenUnsafeGet(LocationSummary* locations, bool is_long, +static void GenUnsafeGet(LocationSummary* locations, Primitive::Type type, bool is_volatile ATTRIBUTE_UNUSED, X86_64Assembler* assembler) { CpuRegister base = locations->InAt(1).AsRegister<CpuRegister>(); CpuRegister offset = locations->InAt(2).AsRegister<CpuRegister>(); CpuRegister trg = locations->Out().AsRegister<CpuRegister>(); - if (is_long) { - __ movq(trg, Address(base, offset, ScaleFactor::TIMES_1, 0)); - } else { - // TODO: Distinguish object. In case we move to an actual compressed heap, retrieving an object - // pointer will entail an unpack operation. - __ movl(trg, Address(base, offset, ScaleFactor::TIMES_1, 0)); + switch (type) { + case Primitive::kPrimInt: + case Primitive::kPrimNot: + __ movl(trg, Address(base, offset, ScaleFactor::TIMES_1, 0)); + break; + + case Primitive::kPrimLong: + __ movq(trg, Address(base, offset, ScaleFactor::TIMES_1, 0)); + break; + + default: + LOG(FATAL) << "Unsupported op size " << type; + UNREACHABLE(); } } @@ -822,10 +824,10 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke LocationSummary* locations = new (arena) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); - locations->SetOut(Location::SameAsFirstInput()); + locations->SetOut(Location::RequiresRegister()); } void IntrinsicLocationsBuilderX86_64::VisitUnsafeGet(HInvoke* invoke) { @@ -840,19 +842,33 @@ void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLong(HInvoke* invoke) { void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { CreateIntIntIntToIntLocations(arena_, invoke); } +void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObject(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke); +} + void IntrinsicCodeGeneratorX86_64::VisitUnsafeGet(HInvoke* invoke) { - GenUnsafeGet(invoke->GetLocations(), false, false, GetAssembler()); + GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimInt, false, GetAssembler()); } void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke->GetLocations(), false, true, GetAssembler()); + GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimInt, true, GetAssembler()); } void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLong(HInvoke* invoke) { - GenUnsafeGet(invoke->GetLocations(), true, false, GetAssembler()); + GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimLong, false, GetAssembler()); } void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke->GetLocations(), true, true, GetAssembler()); + GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimLong, true, GetAssembler()); } +void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObject(HInvoke* invoke) { + GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimNot, false, GetAssembler()); +} +void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { + GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimNot, true, GetAssembler()); +} + static void CreateIntIntIntIntToVoidPlusTempsLocations(ArenaAllocator* arena, Primitive::Type type, @@ -860,7 +876,7 @@ static void CreateIntIntIntIntToVoidPlusTempsLocations(ArenaAllocator* arena, LocationSummary* locations = new (arena) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::NoLocation()); + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); locations->SetInAt(3, Location::RequiresRegister()); diff --git a/compiler/optimizing/intrinsics_x86_64.h b/compiler/optimizing/intrinsics_x86_64.h index c1fa99c2dc..dfae7fa90e 100644 --- a/compiler/optimizing/intrinsics_x86_64.h +++ b/compiler/optimizing/intrinsics_x86_64.h @@ -70,7 +70,7 @@ INTRINSICS_LIST(OPTIMIZING_INTRINSICS) private: X86_64Assembler* GetAssembler(); - ArenaAllocator* GetArena(); + ArenaAllocator* GetAllocator(); CodeGeneratorX86_64* codegen_; diff --git a/compiler/optimizing/locations.h b/compiler/optimizing/locations.h index 6bf8f776fd..8b06d607f5 100644 --- a/compiler/optimizing/locations.h +++ b/compiler/optimizing/locations.h @@ -446,6 +446,8 @@ class RegisterSet : public ValueObject { DISALLOW_COPY_AND_ASSIGN(RegisterSet); }; +static constexpr bool kIntrinsified = true; + /** * The code generator computes LocationSummary for each instruction so that * the instruction itself knows what code to generate: where to find the inputs diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index 2cc021cccf..3e4028ed7f 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -847,6 +847,10 @@ class HInstruction : public ArenaObject<kArenaAllocMisc> { virtual bool CanThrow() const { return false; } bool HasSideEffects() const { return side_effects_.HasSideEffects(); } + // Does not apply for all instructions, but having this at top level greatly + // simplifies the null check elimination. + virtual bool CanBeNull() const { return true; } + virtual bool CanDoImplicitNullCheck() const { return false; } void AddUseAt(HInstruction* user, size_t index) { @@ -1711,9 +1715,11 @@ class HInvokeStaticOrDirect : public HInvoke { Primitive::Type return_type, uint32_t dex_pc, uint32_t dex_method_index, + bool is_recursive, InvokeType invoke_type) : HInvoke(arena, number_of_arguments, return_type, dex_pc, dex_method_index), - invoke_type_(invoke_type) {} + invoke_type_(invoke_type), + is_recursive_(is_recursive) {} bool CanDoImplicitNullCheck() const OVERRIDE { // We access the method via the dex cache so we can't do an implicit null check. @@ -1722,11 +1728,13 @@ class HInvokeStaticOrDirect : public HInvoke { } InvokeType GetInvokeType() const { return invoke_type_; } + bool IsRecursive() const { return is_recursive_; } DECLARE_INSTRUCTION(InvokeStaticOrDirect); private: const InvokeType invoke_type_; + const bool is_recursive_; DISALLOW_COPY_AND_ASSIGN(HInvokeStaticOrDirect); }; @@ -1802,6 +1810,8 @@ class HNewInstance : public HExpression<0> { // TODO: optimize when possible. bool CanThrow() const OVERRIDE { return true; } + bool CanBeNull() const OVERRIDE { return false; } + DECLARE_INSTRUCTION(NewInstance); private: @@ -1838,7 +1848,9 @@ class HNewArray : public HExpression<1> { uint16_t GetTypeIndex() const { return type_index_; } // Calls runtime so needs an environment. - virtual bool NeedsEnvironment() const { return true; } + bool NeedsEnvironment() const OVERRIDE { return true; } + + bool CanBeNull() const OVERRIDE { return false; } DECLARE_INSTRUCTION(NewArray); @@ -2088,18 +2100,23 @@ class HXor : public HBinaryOperation { // the calling convention. class HParameterValue : public HExpression<0> { public: - HParameterValue(uint8_t index, Primitive::Type parameter_type) - : HExpression(parameter_type, SideEffects::None()), index_(index) {} + HParameterValue(uint8_t index, Primitive::Type parameter_type, bool is_this = false) + : HExpression(parameter_type, SideEffects::None()), index_(index), is_this_(is_this) {} uint8_t GetIndex() const { return index_; } + bool CanBeNull() const OVERRIDE { return !is_this_; } + DECLARE_INSTRUCTION(ParameterValue); private: // The index of this parameter in the parameters list. Must be less - // than HGraph::number_of_in_vregs_; + // than HGraph::number_of_in_vregs_. const uint8_t index_; + // Whether or not the parameter value corresponds to 'this' argument. + const bool is_this_; + DISALLOW_COPY_AND_ASSIGN(HParameterValue); }; @@ -2158,22 +2175,26 @@ class HPhi : public HInstruction { inputs_(arena, number_of_inputs), reg_number_(reg_number), type_(type), - is_live_(false) { + is_live_(false), + can_be_null_(true) { inputs_.SetSize(number_of_inputs); } - virtual size_t InputCount() const { return inputs_.Size(); } - virtual HInstruction* InputAt(size_t i) const { return inputs_.Get(i); } + size_t InputCount() const OVERRIDE { return inputs_.Size(); } + HInstruction* InputAt(size_t i) const OVERRIDE { return inputs_.Get(i); } - virtual void SetRawInputAt(size_t index, HInstruction* input) { + void SetRawInputAt(size_t index, HInstruction* input) OVERRIDE { inputs_.Put(index, input); } void AddInput(HInstruction* input); - virtual Primitive::Type GetType() const { return type_; } + Primitive::Type GetType() const OVERRIDE { return type_; } void SetType(Primitive::Type type) { type_ = type; } + bool CanBeNull() const OVERRIDE { return can_be_null_; } + void SetCanBeNull(bool can_be_null) { can_be_null_ = can_be_null; } + uint32_t GetRegNumber() const { return reg_number_; } void SetDead() { is_live_ = false; } @@ -2188,6 +2209,7 @@ class HPhi : public HInstruction { const uint32_t reg_number_; Primitive::Type type_; bool is_live_; + bool can_be_null_; DISALLOW_COPY_AND_ASSIGN(HPhi); }; @@ -2199,15 +2221,17 @@ class HNullCheck : public HExpression<1> { SetRawInputAt(0, value); } - virtual bool CanBeMoved() const { return true; } - virtual bool InstructionDataEquals(HInstruction* other) const { + bool CanBeMoved() const OVERRIDE { return true; } + bool InstructionDataEquals(HInstruction* other) const OVERRIDE { UNUSED(other); return true; } - virtual bool NeedsEnvironment() const { return true; } + bool NeedsEnvironment() const OVERRIDE { return true; } - virtual bool CanThrow() const { return true; } + bool CanThrow() const OVERRIDE { return true; } + + bool CanBeNull() const OVERRIDE { return false; } uint32_t GetDexPc() const { return dex_pc_; } diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 4f05afa1a6..705345b279 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -43,6 +43,7 @@ #include "ssa_builder.h" #include "ssa_phi_elimination.h" #include "ssa_liveness_analysis.h" +#include "reference_type_propagation.h" #include "utils/arena_allocator.h" namespace art { @@ -196,6 +197,18 @@ static bool CanOptimize(const DexFile::CodeItem& code_item) { return code_item.tries_size_ == 0; } +static void RunOptimizations(HOptimization* optimizations[], + size_t length, + const HGraphVisualizer& visualizer) { + for (size_t i = 0; i < length; ++i) { + HOptimization* optimization = optimizations[i]; + visualizer.DumpGraph(optimization->GetPassName(), /*is_after=*/false); + optimization->Run(); + visualizer.DumpGraph(optimization->GetPassName(), /*is_after=*/true); + optimization->Check(); + } +} + static void RunOptimizations(HGraph* graph, CompilerDriver* driver, OptimizingCompilerStats* stats, @@ -213,7 +226,8 @@ static void RunOptimizations(HGraph* graph, SideEffectsAnalysis side_effects(graph); GVNOptimization gvn(graph, side_effects); BoundsCheckElimination bce(graph); - InstructionSimplifier simplify2(graph); + ReferenceTypePropagation type_propagation(graph); + InstructionSimplifier simplify2(graph, "instruction_simplifier_after_types"); IntrinsicsRecognizer intrinsics(graph, dex_compilation_unit.GetDexFile(), driver); @@ -229,16 +243,11 @@ static void RunOptimizations(HGraph* graph, &side_effects, &gvn, &bce, + &type_propagation, &simplify2 }; - for (size_t i = 0; i < arraysize(optimizations); ++i) { - HOptimization* optimization = optimizations[i]; - visualizer.DumpGraph(optimization->GetPassName(), /*is_after=*/false); - optimization->Run(); - visualizer.DumpGraph(optimization->GetPassName(), /*is_after=*/true); - optimization->Check(); - } + RunOptimizations(optimizations, arraysize(optimizations), visualizer); } // The stack map we generate must be 4-byte aligned on ARM. Since existing diff --git a/compiler/optimizing/optimizing_unit_test.h b/compiler/optimizing/optimizing_unit_test.h index b3eb1e2d51..29d47e13cc 100644 --- a/compiler/optimizing/optimizing_unit_test.h +++ b/compiler/optimizing/optimizing_unit_test.h @@ -19,6 +19,7 @@ #include "nodes.h" #include "builder.h" +#include "compiler/dex/pass_manager.h" #include "dex_file.h" #include "dex_instruction.h" #include "ssa_liveness_analysis.h" diff --git a/compiler/optimizing/ssa_type_propagation.cc b/compiler/optimizing/primitive_type_propagation.cc index 947427b67f..7e274f6ebf 100644 --- a/compiler/optimizing/ssa_type_propagation.cc +++ b/compiler/optimizing/primitive_type_propagation.cc @@ -14,10 +14,10 @@ * limitations under the License. */ -#include "ssa_builder.h" -#include "ssa_type_propagation.h" +#include "primitive_type_propagation.h" #include "nodes.h" +#include "ssa_builder.h" namespace art { @@ -39,7 +39,7 @@ static Primitive::Type MergeTypes(Primitive::Type existing, Primitive::Type new_ // Re-compute and update the type of the instruction. Returns // whether or not the type was changed. -bool SsaTypePropagation::UpdateType(HPhi* phi) { +bool PrimitiveTypePropagation::UpdateType(HPhi* phi) { Primitive::Type existing = phi->GetType(); Primitive::Type new_type = existing; @@ -67,14 +67,14 @@ bool SsaTypePropagation::UpdateType(HPhi* phi) { return existing != new_type; } -void SsaTypePropagation::Run() { +void PrimitiveTypePropagation::Run() { for (HReversePostOrderIterator it(*graph_); !it.Done(); it.Advance()) { VisitBasicBlock(it.Current()); } ProcessWorklist(); } -void SsaTypePropagation::VisitBasicBlock(HBasicBlock* block) { +void PrimitiveTypePropagation::VisitBasicBlock(HBasicBlock* block) { if (block->IsLoopHeader()) { for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { HPhi* phi = it.Current()->AsPhi(); @@ -100,7 +100,7 @@ void SsaTypePropagation::VisitBasicBlock(HBasicBlock* block) { } } -void SsaTypePropagation::ProcessWorklist() { +void PrimitiveTypePropagation::ProcessWorklist() { while (!worklist_.IsEmpty()) { HPhi* instruction = worklist_.Pop(); if (UpdateType(instruction)) { @@ -109,11 +109,11 @@ void SsaTypePropagation::ProcessWorklist() { } } -void SsaTypePropagation::AddToWorklist(HPhi* instruction) { +void PrimitiveTypePropagation::AddToWorklist(HPhi* instruction) { worklist_.Add(instruction); } -void SsaTypePropagation::AddDependentInstructionsToWorklist(HPhi* instruction) { +void PrimitiveTypePropagation::AddDependentInstructionsToWorklist(HPhi* instruction) { for (HUseIterator<HInstruction*> it(instruction->GetUses()); !it.Done(); it.Advance()) { HPhi* phi = it.Current()->GetUser()->AsPhi(); if (phi != nullptr) { diff --git a/compiler/optimizing/ssa_type_propagation.h b/compiler/optimizing/primitive_type_propagation.h index f4d3d6344a..1374cbb6ab 100644 --- a/compiler/optimizing/ssa_type_propagation.h +++ b/compiler/optimizing/primitive_type_propagation.h @@ -14,17 +14,17 @@ * limitations under the License. */ -#ifndef ART_COMPILER_OPTIMIZING_SSA_TYPE_PROPAGATION_H_ -#define ART_COMPILER_OPTIMIZING_SSA_TYPE_PROPAGATION_H_ +#ifndef ART_COMPILER_OPTIMIZING_PRIMITIVE_TYPE_PROPAGATION_H_ +#define ART_COMPILER_OPTIMIZING_PRIMITIVE_TYPE_PROPAGATION_H_ #include "nodes.h" namespace art { -// Compute and propagate types of phis in the graph. -class SsaTypePropagation : public ValueObject { +// Compute and propagate primitive types of phis in the graph. +class PrimitiveTypePropagation : public ValueObject { public: - explicit SsaTypePropagation(HGraph* graph) + explicit PrimitiveTypePropagation(HGraph* graph) : graph_(graph), worklist_(graph->GetArena(), kDefaultWorklistSize) {} void Run(); @@ -41,9 +41,9 @@ class SsaTypePropagation : public ValueObject { static constexpr size_t kDefaultWorklistSize = 8; - DISALLOW_COPY_AND_ASSIGN(SsaTypePropagation); + DISALLOW_COPY_AND_ASSIGN(PrimitiveTypePropagation); }; } // namespace art -#endif // ART_COMPILER_OPTIMIZING_SSA_TYPE_PROPAGATION_H_ +#endif // ART_COMPILER_OPTIMIZING_PRIMITIVE_TYPE_PROPAGATION_H_ diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc new file mode 100644 index 0000000000..24e6837f45 --- /dev/null +++ b/compiler/optimizing/reference_type_propagation.cc @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "reference_type_propagation.h" + +namespace art { + +// TODO: Only do the analysis on reference types. We currently have to handle +// the `null` constant, that is represented as a `HIntConstant` and therefore +// has the Primitive::kPrimInt type. + +void ReferenceTypePropagation::Run() { + // Compute null status for instructions. + + // To properly propagate not-null info we need to visit in the dominator-based order. + // Reverse post order guarantees a node's dominators are visited first. + // We take advantage of this order in `VisitBasicBlock`. + for (HReversePostOrderIterator it(*graph_); !it.Done(); it.Advance()) { + VisitBasicBlock(it.Current()); + } + ProcessWorklist(); +} + +// Re-computes and updates the nullability of the instruction. Returns whether or +// not the nullability was changed. +bool ReferenceTypePropagation::UpdateNullability(HPhi* phi) { + bool existing_can_be_null = phi->CanBeNull(); + bool new_can_be_null = false; + for (size_t i = 0; i < phi->InputCount(); i++) { + new_can_be_null |= phi->InputAt(i)->CanBeNull(); + } + phi->SetCanBeNull(new_can_be_null); + + return existing_can_be_null != new_can_be_null; +} + + +void ReferenceTypePropagation::VisitBasicBlock(HBasicBlock* block) { + if (block->IsLoopHeader()) { + for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { + // Set the initial type for the phi. Use the non back edge input for reaching + // a fixed point faster. + HPhi* phi = it.Current()->AsPhi(); + AddToWorklist(phi); + phi->SetCanBeNull(phi->InputAt(0)->CanBeNull()); + } + } else { + for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { + // Eagerly compute the type of the phi, for quicker convergence. Note + // that we don't need to add users to the worklist because we are + // doing a reverse post-order visit, therefore either the phi users are + // non-loop phi and will be visited later in the visit, or are loop-phis, + // and they are already in the work list. + UpdateNullability(it.Current()->AsPhi()); + } + } +} + +void ReferenceTypePropagation::ProcessWorklist() { + while (!worklist_.IsEmpty()) { + HPhi* instruction = worklist_.Pop(); + if (UpdateNullability(instruction)) { + AddDependentInstructionsToWorklist(instruction); + } + } +} + +void ReferenceTypePropagation::AddToWorklist(HPhi* instruction) { + worklist_.Add(instruction); +} + +void ReferenceTypePropagation::AddDependentInstructionsToWorklist(HPhi* instruction) { + for (HUseIterator<HInstruction*> it(instruction->GetUses()); !it.Done(); it.Advance()) { + HPhi* phi = it.Current()->GetUser()->AsPhi(); + if (phi != nullptr) { + AddToWorklist(phi); + } + } +} + +} // namespace art diff --git a/compiler/optimizing/reference_type_propagation.h b/compiler/optimizing/reference_type_propagation.h new file mode 100644 index 0000000000..a74319d0c5 --- /dev/null +++ b/compiler/optimizing/reference_type_propagation.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_REFERENCE_TYPE_PROPAGATION_H_ +#define ART_COMPILER_OPTIMIZING_REFERENCE_TYPE_PROPAGATION_H_ + +#include "nodes.h" +#include "optimization.h" + +namespace art { + +/** + * Propagates reference types to instructions. + * TODO: Currently only nullability is computed. + */ +class ReferenceTypePropagation : public HOptimization { + public: + explicit ReferenceTypePropagation(HGraph* graph) + : HOptimization(graph, true, "reference_type_propagation"), + worklist_(graph->GetArena(), kDefaultWorklistSize) {} + + void Run() OVERRIDE; + + private: + void VisitBasicBlock(HBasicBlock* block); + void ProcessWorklist(); + void AddToWorklist(HPhi* phi); + void AddDependentInstructionsToWorklist(HPhi* phi); + bool UpdateNullability(HPhi* phi); + + GrowableArray<HPhi*> worklist_; + + static constexpr size_t kDefaultWorklistSize = 8; + + DISALLOW_COPY_AND_ASSIGN(ReferenceTypePropagation); +}; + +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_REFERENCE_TYPE_PROPAGATION_H_ diff --git a/compiler/optimizing/register_allocator.cc b/compiler/optimizing/register_allocator.cc index 260076acbf..6f8f688a2b 100644 --- a/compiler/optimizing/register_allocator.cc +++ b/compiler/optimizing/register_allocator.cc @@ -83,12 +83,8 @@ bool RegisterAllocator::CanAllocateRegistersFor(const HGraph& graph, !it.Done(); it.Advance()) { HInstruction* current = it.Current(); - if (instruction_set == kX86) { - if (current->GetType() == Primitive::kPrimLong || - current->GetType() == Primitive::kPrimFloat || - current->GetType() == Primitive::kPrimDouble) { - return false; - } + if (instruction_set == kX86 && current->GetType() == Primitive::kPrimLong) { + return false; } } } diff --git a/compiler/optimizing/ssa_builder.cc b/compiler/optimizing/ssa_builder.cc index 4f9c3b87c1..c9a21aa681 100644 --- a/compiler/optimizing/ssa_builder.cc +++ b/compiler/optimizing/ssa_builder.cc @@ -17,7 +17,7 @@ #include "ssa_builder.h" #include "nodes.h" -#include "ssa_type_propagation.h" +#include "primitive_type_propagation.h" #include "ssa_phi_elimination.h" namespace art { @@ -52,7 +52,7 @@ void SsaBuilder::BuildSsa() { // 4) Propagate types of phis. At this point, phis are typed void in the general // case, or float or double when we created a floating-point equivalent. So we // need to propagate the types across phis to give them a correct type. - SsaTypePropagation type_propagation(GetGraph()); + PrimitiveTypePropagation type_propagation(GetGraph()); type_propagation.Run(); // 5) Clear locals. |