diff options
Diffstat (limited to 'compiler/optimizing')
| -rw-r--r-- | compiler/optimizing/bounds_check_elimination.cc | 33 | ||||
| -rw-r--r-- | compiler/optimizing/inliner.cc | 69 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics.h | 40 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_arm.cc | 79 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_arm64.cc | 312 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_list.h | 2 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_mips.cc | 705 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_mips64.cc | 229 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_x86.cc | 50 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_x86_64.cc | 37 | ||||
| -rw-r--r-- | compiler/optimizing/nodes.cc | 170 | ||||
| -rw-r--r-- | compiler/optimizing/nodes.h | 8 | ||||
| -rw-r--r-- | compiler/optimizing/optimizing_compiler.cc | 8 | ||||
| -rw-r--r-- | compiler/optimizing/prepare_for_register_allocation.h | 3 |
14 files changed, 961 insertions, 784 deletions
diff --git a/compiler/optimizing/bounds_check_elimination.cc b/compiler/optimizing/bounds_check_elimination.cc index a7a1c0f2c4..f2929bcc18 100644 --- a/compiler/optimizing/bounds_check_elimination.cc +++ b/compiler/optimizing/bounds_check_elimination.cc @@ -533,6 +533,8 @@ class BCEVisitor : public HGraphVisitor { first_index_bounds_check_map_( std::less<int>(), graph->GetArena()->Adapter(kArenaAllocBoundsCheckElimination)), + dynamic_bce_standby_( + graph->GetArena()->Adapter(kArenaAllocBoundsCheckElimination)), early_exit_loop_( std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocBoundsCheckElimination)), @@ -553,6 +555,13 @@ class BCEVisitor : public HGraphVisitor { } void Finish() { + // Retry dynamic bce candidates on standby that are still in the graph. + for (HBoundsCheck* bounds_check : dynamic_bce_standby_) { + if (bounds_check->IsInBlock()) { + TryDynamicBCE(bounds_check); + } + } + // Preserve SSA structure which may have been broken by adding one or more // new taken-test structures (see TransformLoopForDeoptimizationIfNeeded()). InsertPhiNodes(); @@ -561,6 +570,7 @@ class BCEVisitor : public HGraphVisitor { early_exit_loop_.clear(); taken_test_loop_.clear(); finite_loop_.clear(); + dynamic_bce_standby_.clear(); } private: @@ -1301,7 +1311,7 @@ class BCEVisitor : public HGraphVisitor { if (DynamicBCESeemsProfitable(loop, instruction->GetBlock()) && induction_range_.CanGenerateCode( instruction, index, &needs_finite_test, &needs_taken_test) && - CanHandleInfiniteLoop(loop, index, needs_finite_test) && + CanHandleInfiniteLoop(loop, instruction, index, needs_finite_test) && CanHandleLength(loop, length, needs_taken_test)) { // do this test last (may code gen) HInstruction* lower = nullptr; HInstruction* upper = nullptr; @@ -1433,7 +1443,7 @@ class BCEVisitor : public HGraphVisitor { * ensure the loop is finite. */ bool CanHandleInfiniteLoop( - HLoopInformation* loop, HInstruction* index, bool needs_infinite_test) { + HLoopInformation* loop, HBoundsCheck* check, HInstruction* index, bool needs_infinite_test) { if (needs_infinite_test) { // If we already forced the loop to be finite, allow directly. const uint32_t loop_id = loop->GetHeader()->GetBlockId(); @@ -1455,6 +1465,9 @@ class BCEVisitor : public HGraphVisitor { } } } + // If bounds check made it this far, it is worthwhile to check later if + // the loop was forced finite by another candidate. + dynamic_bce_standby_.push_back(check); return false; } return true; @@ -1676,6 +1689,9 @@ class BCEVisitor : public HGraphVisitor { // in a block that checks an index against that HArrayLength. ArenaSafeMap<int, HBoundsCheck*> first_index_bounds_check_map_; + // Stand by list for dynamic bce. + ArenaVector<HBoundsCheck*> dynamic_bce_standby_; + // Early-exit loop bookkeeping. ArenaSafeMap<uint32_t, bool> early_exit_loop_; @@ -1711,21 +1727,18 @@ void BoundsCheckElimination::Run() { // that value dominated by that instruction fits in that range. Range of that // value can be narrowed further down in the dominator tree. BCEVisitor visitor(graph_, side_effects_, induction_analysis_); - HBasicBlock* last_visited_block = nullptr; for (HReversePostOrderIterator it(*graph_); !it.Done(); it.Advance()) { HBasicBlock* current = it.Current(); - if (current == last_visited_block) { - // We may insert blocks into the reverse post order list when processing - // a loop header. Don't process it again. - DCHECK(current->IsLoopHeader()); - continue; - } if (visitor.IsAddedBlock(current)) { // Skip added blocks. Their effects are already taken care of. continue; } visitor.VisitBasicBlock(current); - last_visited_block = current; + // Skip forward to the current block in case new basic blocks were inserted + // (which always appear earlier in reverse post order) to avoid visiting the + // same basic block twice. + for ( ; !it.Done() && it.Current() != current; it.Advance()) { + } } // Perform cleanup. diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index d55009554f..3e3719e6ea 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -562,31 +562,16 @@ void HInliner::CreateDiamondPatternForPolymorphicInline(HInstruction* compare, graph_->reverse_post_order_[++index] = otherwise; graph_->reverse_post_order_[++index] = merge; - // Set the loop information of the newly created blocks. - HLoopInformation* loop_info = cursor_block->GetLoopInformation(); - if (loop_info != nullptr) { - then->SetLoopInformation(cursor_block->GetLoopInformation()); - merge->SetLoopInformation(cursor_block->GetLoopInformation()); - otherwise->SetLoopInformation(cursor_block->GetLoopInformation()); - for (HLoopInformationOutwardIterator loop_it(*cursor_block); - !loop_it.Done(); - loop_it.Advance()) { - loop_it.Current()->Add(then); - loop_it.Current()->Add(merge); - loop_it.Current()->Add(otherwise); - } - // In case the original invoke location was a back edge, we need to update - // the loop to now have the merge block as a back edge. - if (loop_info->IsBackEdge(*original_invoke_block)) { - loop_info->RemoveBackEdge(original_invoke_block); - loop_info->AddBackEdge(merge); - } - } - // Set the try/catch information of the newly created blocks. - then->SetTryCatchInformation(cursor_block->GetTryCatchInformation()); - merge->SetTryCatchInformation(cursor_block->GetTryCatchInformation()); - otherwise->SetTryCatchInformation(cursor_block->GetTryCatchInformation()); + graph_->UpdateLoopAndTryInformationOfNewBlock( + then, original_invoke_block, /* replace_if_back_edge */ false); + graph_->UpdateLoopAndTryInformationOfNewBlock( + otherwise, original_invoke_block, /* replace_if_back_edge */ false); + + // In case the original invoke location was a back edge, we need to update + // the loop to now have the merge block as a back edge. + graph_->UpdateLoopAndTryInformationOfNewBlock( + merge, original_invoke_block, /* replace_if_back_edge */ true); } bool HInliner::TryInlinePolymorphicCallToSameTarget(HInvoke* invoke_instruction, @@ -603,6 +588,10 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget(HInvoke* invoke_instruction, DCHECK(resolved_method != nullptr); ArtMethod* actual_method = nullptr; + size_t method_index = invoke_instruction->IsInvokeVirtual() + ? invoke_instruction->AsInvokeVirtual()->GetVTableIndex() + : invoke_instruction->AsInvokeInterface()->GetImtIndex(); + // Check whether we are actually calling the same method among // the different types seen. for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) { @@ -611,13 +600,18 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget(HInvoke* invoke_instruction, } ArtMethod* new_method = nullptr; if (invoke_instruction->IsInvokeInterface()) { - new_method = ic.GetTypeAt(i)->FindVirtualMethodForInterface( - resolved_method, pointer_size); + new_method = ic.GetTypeAt(i)->GetEmbeddedImTableEntry( + method_index % mirror::Class::kImtSize, pointer_size); + if (new_method->IsRuntimeMethod()) { + // Bail out as soon as we see a conflict trampoline in one of the target's + // interface table. + return false; + } } else { DCHECK(invoke_instruction->IsInvokeVirtual()); - new_method = ic.GetTypeAt(i)->FindVirtualMethodForVirtual( - resolved_method, pointer_size); + new_method = ic.GetTypeAt(i)->GetEmbeddedVTableEntry(method_index, pointer_size); } + DCHECK(new_method != nullptr); if (actual_method == nullptr) { actual_method = new_method; } else if (actual_method != new_method) { @@ -641,10 +635,6 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget(HInvoke* invoke_instruction, HInstanceFieldGet* receiver_class = BuildGetReceiverClass( class_linker, receiver, invoke_instruction->GetDexPc()); - size_t method_offset = invoke_instruction->IsInvokeVirtual() - ? actual_method->GetVtableIndex() - : invoke_instruction->AsInvokeInterface()->GetImtIndex(); - Primitive::Type type = Is64BitInstructionSet(graph_->GetInstructionSet()) ? Primitive::kPrimLong : Primitive::kPrimInt; @@ -653,7 +643,7 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget(HInvoke* invoke_instruction, type, invoke_instruction->IsInvokeVirtual() ? HClassTableGet::TableKind::kVTable : HClassTableGet::TableKind::kIMTable, - method_offset, + method_index, invoke_instruction->GetDexPc()); HConstant* constant; @@ -1020,6 +1010,8 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, // at runtime, we change this call as if it was a virtual call. invoke_type = kVirtual; } + + const int32_t caller_instruction_counter = graph_->GetCurrentInstructionId(); HGraph* callee_graph = new (graph_->GetArena()) HGraph( graph_->GetArena(), callee_dex_file, @@ -1029,7 +1021,7 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, invoke_type, graph_->IsDebuggable(), /* osr */ false, - graph_->GetCurrentInstructionId()); + caller_instruction_counter); callee_graph->SetArtMethod(resolved_method); OptimizingCompilerStats inline_stats; @@ -1229,7 +1221,16 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, } number_of_inlined_instructions_ += number_of_instructions; + DCHECK_EQ(caller_instruction_counter, graph_->GetCurrentInstructionId()) + << "No instructions can be added to the outer graph while inner graph is being built"; + + const int32_t callee_instruction_counter = callee_graph->GetCurrentInstructionId(); + graph_->SetCurrentInstructionId(callee_instruction_counter); *return_replacement = callee_graph->InlineInto(graph_, invoke_instruction); + + DCHECK_EQ(callee_instruction_counter, callee_graph->GetCurrentInstructionId()) + << "No instructions can be added to the inner graph during inlining into the outer graph"; + return true; } diff --git a/compiler/optimizing/intrinsics.h b/compiler/optimizing/intrinsics.h index 2ab50bb436..0cec5ccfd3 100644 --- a/compiler/optimizing/intrinsics.h +++ b/compiler/optimizing/intrinsics.h @@ -193,6 +193,46 @@ class SystemArrayCopyOptimizations : public IntrinsicOptimizations { #undef INTRISIC_OPTIMIZATION +// +// Macros for use in the intrinsics code generators. +// + +// Defines an unimplemented intrinsic: that is, a method call that is recognized as an +// intrinsic to exploit e.g. no side-effects or exceptions, but otherwise not handled +// by this architecture-specific intrinsics code generator. Eventually it is implemented +// as a true method call. +#define UNIMPLEMENTED_INTRINSIC(Arch, Name) \ +void IntrinsicLocationsBuilder ## Arch::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ +} \ +void IntrinsicCodeGenerator ## Arch::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ +} + +// Defines a list of unreached intrinsics: that is, method calls that are recognized as +// an intrinsic, and then always converted into HIR instructions before they reach any +// architecture-specific intrinsics code generator. +#define UNREACHABLE_INTRINSIC(Arch, Name) \ +void IntrinsicLocationsBuilder ## Arch::Visit ## Name(HInvoke* invoke) { \ + LOG(FATAL) << "Unreachable: intrinsic " << invoke->GetIntrinsic() \ + << " should have been converted to HIR"; \ +} \ +void IntrinsicCodeGenerator ## Arch::Visit ## Name(HInvoke* invoke) { \ + LOG(FATAL) << "Unreachable: intrinsic " << invoke->GetIntrinsic() \ + << " should have been converted to HIR"; \ +} +#define UNREACHABLE_INTRINSICS(Arch) \ +UNREACHABLE_INTRINSIC(Arch, FloatFloatToIntBits) \ +UNREACHABLE_INTRINSIC(Arch, DoubleDoubleToLongBits) \ +UNREACHABLE_INTRINSIC(Arch, FloatIsNaN) \ +UNREACHABLE_INTRINSIC(Arch, DoubleIsNaN) \ +UNREACHABLE_INTRINSIC(Arch, IntegerRotateLeft) \ +UNREACHABLE_INTRINSIC(Arch, LongRotateLeft) \ +UNREACHABLE_INTRINSIC(Arch, IntegerRotateRight) \ +UNREACHABLE_INTRINSIC(Arch, LongRotateRight) \ +UNREACHABLE_INTRINSIC(Arch, IntegerCompare) \ +UNREACHABLE_INTRINSIC(Arch, LongCompare) \ +UNREACHABLE_INTRINSIC(Arch, IntegerSignum) \ +UNREACHABLE_INTRINSIC(Arch, LongSignum) + } // namespace art #endif // ART_COMPILER_OPTIMIZING_INTRINSICS_H_ diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc index 4ce919ee39..69c970852d 100644 --- a/compiler/optimizing/intrinsics_arm.cc +++ b/compiler/optimizing/intrinsics_arm.cc @@ -1151,6 +1151,7 @@ static void GenerateVisitStringIndexOf(HInvoke* invoke, __ LoadFromOffset(kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pIndexOf).Int32Value()); + CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>(); __ blx(LR); if (slow_path != nullptr) { @@ -1242,6 +1243,12 @@ void IntrinsicLocationsBuilderARM::VisitStringNewStringFromChars(HInvoke* invoke void IntrinsicCodeGeneratorARM::VisitStringNewStringFromChars(HInvoke* invoke) { ArmAssembler* assembler = GetAssembler(); + // No need to emit code checking whether `locations->InAt(2)` is a null + // pointer, as callers of the native method + // + // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) + // + // all include a null check on `data` before calling that method. __ LoadFromOffset( kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromChars).Int32Value()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); @@ -1972,54 +1979,30 @@ void IntrinsicCodeGeneratorARM::VisitStringGetCharsNoCheck(HInvoke* invoke) { __ Bind(&done); } -// Unimplemented intrinsics. - -#define UNIMPLEMENTED_INTRINSIC(Name) \ -void IntrinsicLocationsBuilderARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} \ -void IntrinsicCodeGeneratorARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} - -UNIMPLEMENTED_INTRINSIC(IntegerBitCount) -UNIMPLEMENTED_INTRINSIC(LongBitCount) -UNIMPLEMENTED_INTRINSIC(MathMinDoubleDouble) -UNIMPLEMENTED_INTRINSIC(MathMinFloatFloat) -UNIMPLEMENTED_INTRINSIC(MathMaxDoubleDouble) -UNIMPLEMENTED_INTRINSIC(MathMaxFloatFloat) -UNIMPLEMENTED_INTRINSIC(MathMinLongLong) -UNIMPLEMENTED_INTRINSIC(MathMaxLongLong) -UNIMPLEMENTED_INTRINSIC(MathCeil) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(MathFloor) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(MathRint) -UNIMPLEMENTED_INTRINSIC(MathRoundDouble) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(MathRoundFloat) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(UnsafeCASLong) // High register pressure. -UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar) -UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) - -UNIMPLEMENTED_INTRINSIC(FloatIsInfinite) -UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite) - -UNIMPLEMENTED_INTRINSIC(IntegerHighestOneBit) -UNIMPLEMENTED_INTRINSIC(LongHighestOneBit) -UNIMPLEMENTED_INTRINSIC(IntegerLowestOneBit) -UNIMPLEMENTED_INTRINSIC(LongLowestOneBit) - -// Handled as HIR instructions. -UNIMPLEMENTED_INTRINSIC(FloatFloatToIntBits) -UNIMPLEMENTED_INTRINSIC(DoubleDoubleToLongBits) -UNIMPLEMENTED_INTRINSIC(FloatIsNaN) -UNIMPLEMENTED_INTRINSIC(DoubleIsNaN) -UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft) -UNIMPLEMENTED_INTRINSIC(LongRotateLeft) -UNIMPLEMENTED_INTRINSIC(IntegerRotateRight) -UNIMPLEMENTED_INTRINSIC(LongRotateRight) -UNIMPLEMENTED_INTRINSIC(IntegerCompare) -UNIMPLEMENTED_INTRINSIC(LongCompare) -UNIMPLEMENTED_INTRINSIC(IntegerSignum) -UNIMPLEMENTED_INTRINSIC(LongSignum) - -#undef UNIMPLEMENTED_INTRINSIC +UNIMPLEMENTED_INTRINSIC(ARM, IntegerBitCount) +UNIMPLEMENTED_INTRINSIC(ARM, LongBitCount) +UNIMPLEMENTED_INTRINSIC(ARM, MathMinDoubleDouble) +UNIMPLEMENTED_INTRINSIC(ARM, MathMinFloatFloat) +UNIMPLEMENTED_INTRINSIC(ARM, MathMaxDoubleDouble) +UNIMPLEMENTED_INTRINSIC(ARM, MathMaxFloatFloat) +UNIMPLEMENTED_INTRINSIC(ARM, MathMinLongLong) +UNIMPLEMENTED_INTRINSIC(ARM, MathMaxLongLong) +UNIMPLEMENTED_INTRINSIC(ARM, MathCeil) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARM, MathFloor) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARM, MathRint) +UNIMPLEMENTED_INTRINSIC(ARM, MathRoundDouble) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARM, MathRoundFloat) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARM, UnsafeCASLong) // High register pressure. +UNIMPLEMENTED_INTRINSIC(ARM, SystemArrayCopyChar) +UNIMPLEMENTED_INTRINSIC(ARM, ReferenceGetReferent) +UNIMPLEMENTED_INTRINSIC(ARM, FloatIsInfinite) +UNIMPLEMENTED_INTRINSIC(ARM, DoubleIsInfinite) +UNIMPLEMENTED_INTRINSIC(ARM, IntegerHighestOneBit) +UNIMPLEMENTED_INTRINSIC(ARM, LongHighestOneBit) +UNIMPLEMENTED_INTRINSIC(ARM, IntegerLowestOneBit) +UNIMPLEMENTED_INTRINSIC(ARM, LongLowestOneBit) + +UNREACHABLE_INTRINSICS(ARM) #undef __ diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index 4be1695a94..934b42762e 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -46,6 +46,7 @@ using helpers::RegisterFrom; using helpers::SRegisterFrom; using helpers::WRegisterFrom; using helpers::XRegisterFrom; +using helpers::InputRegisterAt; namespace { @@ -367,6 +368,40 @@ void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) { GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); } +static void GenBitCount(HInvoke* instr, bool is_long, vixl::MacroAssembler* masm) { + DCHECK(instr->GetType() == Primitive::kPrimInt); + DCHECK((is_long && instr->InputAt(0)->GetType() == Primitive::kPrimLong) || + (!is_long && instr->InputAt(0)->GetType() == Primitive::kPrimInt)); + + Location out = instr->GetLocations()->Out(); + UseScratchRegisterScope temps(masm); + + Register src = InputRegisterAt(instr, 0); + Register dst = is_long ? XRegisterFrom(out) : WRegisterFrom(out); + FPRegister fpr = is_long ? temps.AcquireD() : temps.AcquireS(); + + __ Fmov(fpr, src); + __ Cnt(fpr.V8B(), fpr.V8B()); + __ Addv(fpr.B(), fpr.V8B()); + __ Fmov(dst, fpr); +} + +void IntrinsicLocationsBuilderARM64::VisitLongBitCount(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitLongBitCount(HInvoke* invoke) { + GenBitCount(invoke, /* is_long */ true, GetVIXLAssembler()); +} + +void IntrinsicLocationsBuilderARM64::VisitIntegerBitCount(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARM64::VisitIntegerBitCount(HInvoke* invoke) { + GenBitCount(invoke, /* is_long */ false, GetVIXLAssembler()); +} + static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { LocationSummary* locations = new (arena) LocationSummary(invoke, LocationSummary::kNoCall, @@ -1301,6 +1336,7 @@ static void GenerateVisitStringIndexOf(HInvoke* invoke, } __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value())); + CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>(); __ Blr(lr); if (slow_path != nullptr) { @@ -1392,6 +1428,12 @@ void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invo void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) { vixl::MacroAssembler* masm = GetVIXLAssembler(); + // No need to emit code checking whether `locations->InAt(2)` is a null + // pointer, as callers of the native method + // + // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) + // + // all include a null check on `data` before calling that method. __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value())); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); @@ -1665,43 +1707,253 @@ void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) { __ Bind(&done); } -// Unimplemented intrinsics. +// Mirrors ARRAYCOPY_SHORT_CHAR_ARRAY_THRESHOLD in libcore, so we can choose to use the native +// implementation there for longer copy lengths. +static constexpr int32_t kSystemArrayCopyThreshold = 32; + +static void SetSystemArrayCopyLocationRequires(LocationSummary* locations, + uint32_t at, + HInstruction* input) { + HIntConstant* const_input = input->AsIntConstant(); + if (const_input != nullptr && !vixl::Assembler::IsImmAddSub(const_input->GetValue())) { + locations->SetInAt(at, Location::RequiresRegister()); + } else { + locations->SetInAt(at, Location::RegisterOrConstant(input)); + } +} + +void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopyChar(HInvoke* invoke) { + // Check to see if we have known failures that will cause us to have to bail out + // to the runtime, and just generate the runtime call directly. + HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); + HIntConstant* dst_pos = invoke->InputAt(3)->AsIntConstant(); + + // The positions must be non-negative. + if ((src_pos != nullptr && src_pos->GetValue() < 0) || + (dst_pos != nullptr && dst_pos->GetValue() < 0)) { + // We will have to fail anyways. + return; + } + + // The length must be >= 0 and not so long that we would (currently) prefer libcore's + // native implementation. + HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); + if (length != nullptr) { + int32_t len = length->GetValue(); + if (len < 0 || len > kSystemArrayCopyThreshold) { + // Just call as normal. + return; + } + } + + ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena(); + LocationSummary* locations = new (allocator) LocationSummary(invoke, + LocationSummary::kCallOnSlowPath, + kIntrinsified); + // arraycopy(char[] src, int src_pos, char[] dst, int dst_pos, int length). + locations->SetInAt(0, Location::RequiresRegister()); + SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1)); + locations->SetInAt(2, Location::RequiresRegister()); + SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3)); + SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4)); + + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); +} + +static void CheckSystemArrayCopyPosition(vixl::MacroAssembler* masm, + const Location& pos, + const Register& input, + const Location& length, + SlowPathCodeARM64* slow_path, + const Register& input_len, + const Register& temp, + bool length_is_input_length = false) { + const int32_t length_offset = mirror::Array::LengthOffset().Int32Value(); + if (pos.IsConstant()) { + int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); + if (pos_const == 0) { + if (!length_is_input_length) { + // Check that length(input) >= length. + __ Ldr(temp, MemOperand(input, length_offset)); + __ Cmp(temp, OperandFrom(length, Primitive::kPrimInt)); + __ B(slow_path->GetEntryLabel(), lt); + } + } else { + // Check that length(input) >= pos. + __ Ldr(input_len, MemOperand(input, length_offset)); + __ Subs(temp, input_len, pos_const); + __ B(slow_path->GetEntryLabel(), lt); + + // Check that (length(input) - pos) >= length. + __ Cmp(temp, OperandFrom(length, Primitive::kPrimInt)); + __ B(slow_path->GetEntryLabel(), lt); + } + } else if (length_is_input_length) { + // The only way the copy can succeed is if pos is zero. + __ Cbnz(WRegisterFrom(pos), slow_path->GetEntryLabel()); + } else { + // Check that pos >= 0. + Register pos_reg = WRegisterFrom(pos); + __ Tbnz(pos_reg, pos_reg.size() - 1, slow_path->GetEntryLabel()); + + // Check that pos <= length(input) && (length(input) - pos) >= length. + __ Ldr(temp, MemOperand(input, length_offset)); + __ Subs(temp, temp, pos_reg); + // Ccmp if length(input) >= pos, else definitely bail to slow path (N!=V == lt). + __ Ccmp(temp, OperandFrom(length, Primitive::kPrimInt), NFlag, ge); + __ B(slow_path->GetEntryLabel(), lt); + } +} + +// Compute base source address, base destination address, and end source address +// for System.arraycopy* intrinsics. +static void GenSystemArrayCopyAddresses(vixl::MacroAssembler* masm, + Primitive::Type type, + const Register& src, + const Location& src_pos, + const Register& dst, + const Location& dst_pos, + const Location& copy_length, + const Register& src_base, + const Register& dst_base, + const Register& src_end) { + DCHECK(type == Primitive::kPrimNot || type == Primitive::kPrimChar) + << "Unexpected element type: " + << type; + const int32_t char_size = Primitive::ComponentSize(type); + const int32_t char_size_shift = Primitive::ComponentSizeShift(type); + + uint32_t offset = mirror::Array::DataOffset(char_size).Uint32Value(); + if (src_pos.IsConstant()) { + int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); + __ Add(src_base, src, char_size * constant + offset); + } else { + __ Add(src_base, src, offset); + __ Add(src_base, + src_base, + Operand(XRegisterFrom(src_pos), LSL, char_size_shift)); + } + + if (dst_pos.IsConstant()) { + int32_t constant = dst_pos.GetConstant()->AsIntConstant()->GetValue(); + __ Add(dst_base, dst, char_size * constant + offset); + } else { + __ Add(dst_base, dst, offset); + __ Add(dst_base, + dst_base, + Operand(XRegisterFrom(dst_pos), LSL, char_size_shift)); + } -#define UNIMPLEMENTED_INTRINSIC(Name) \ -void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} \ -void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ + if (copy_length.IsConstant()) { + int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue(); + __ Add(src_end, src_base, char_size * constant); + } else { + __ Add(src_end, + src_base, + Operand(XRegisterFrom(copy_length), LSL, char_size_shift)); + } } -UNIMPLEMENTED_INTRINSIC(IntegerBitCount) -UNIMPLEMENTED_INTRINSIC(LongBitCount) -UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar) -UNIMPLEMENTED_INTRINSIC(SystemArrayCopy) -UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) +void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopyChar(HInvoke* invoke) { + vixl::MacroAssembler* masm = GetVIXLAssembler(); + LocationSummary* locations = invoke->GetLocations(); + Register src = XRegisterFrom(locations->InAt(0)); + Location src_pos = locations->InAt(1); + Register dst = XRegisterFrom(locations->InAt(2)); + Location dst_pos = locations->InAt(3); + Location length = locations->InAt(4); + + SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); + codegen_->AddSlowPath(slow_path); + + // If source and destination are the same, take the slow path. Overlapping copy regions must be + // copied in reverse and we can't know in all cases if it's needed. + __ Cmp(src, dst); + __ B(slow_path->GetEntryLabel(), eq); + + // Bail out if the source is null. + __ Cbz(src, slow_path->GetEntryLabel()); + + // Bail out if the destination is null. + __ Cbz(dst, slow_path->GetEntryLabel()); + + if (!length.IsConstant()) { + // If the length is negative, bail out. + __ Tbnz(WRegisterFrom(length), kWRegSize - 1, slow_path->GetEntryLabel()); + // If the length > 32 then (currently) prefer libcore's native implementation. + __ Cmp(WRegisterFrom(length), kSystemArrayCopyThreshold); + __ B(slow_path->GetEntryLabel(), gt); + } else { + // We have already checked in the LocationsBuilder for the constant case. + DCHECK_GE(length.GetConstant()->AsIntConstant()->GetValue(), 0); + DCHECK_LE(length.GetConstant()->AsIntConstant()->GetValue(), 32); + } -UNIMPLEMENTED_INTRINSIC(FloatIsInfinite) -UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite) + Register src_curr_addr = WRegisterFrom(locations->GetTemp(0)); + Register dst_curr_addr = WRegisterFrom(locations->GetTemp(1)); + Register src_stop_addr = WRegisterFrom(locations->GetTemp(2)); + + CheckSystemArrayCopyPosition(masm, + src_pos, + src, + length, + slow_path, + src_curr_addr, + dst_curr_addr, + false); + + CheckSystemArrayCopyPosition(masm, + dst_pos, + dst, + length, + slow_path, + src_curr_addr, + dst_curr_addr, + false); + + src_curr_addr = src_curr_addr.X(); + dst_curr_addr = dst_curr_addr.X(); + src_stop_addr = src_stop_addr.X(); + + GenSystemArrayCopyAddresses(masm, + Primitive::kPrimChar, + src, + src_pos, + dst, + dst_pos, + length, + src_curr_addr, + dst_curr_addr, + src_stop_addr); + + // Iterate over the arrays and do a raw copy of the chars. + const int32_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); + UseScratchRegisterScope temps(masm); + Register tmp = temps.AcquireW(); + vixl::Label loop, done; + __ Bind(&loop); + __ Cmp(src_curr_addr, src_stop_addr); + __ B(&done, eq); + __ Ldrh(tmp, MemOperand(src_curr_addr, char_size, vixl::PostIndex)); + __ Strh(tmp, MemOperand(dst_curr_addr, char_size, vixl::PostIndex)); + __ B(&loop); + __ Bind(&done); -UNIMPLEMENTED_INTRINSIC(IntegerHighestOneBit) -UNIMPLEMENTED_INTRINSIC(LongHighestOneBit) -UNIMPLEMENTED_INTRINSIC(IntegerLowestOneBit) -UNIMPLEMENTED_INTRINSIC(LongLowestOneBit) + __ Bind(slow_path->GetExitLabel()); +} -// Handled as HIR instructions. -UNIMPLEMENTED_INTRINSIC(FloatFloatToIntBits) -UNIMPLEMENTED_INTRINSIC(DoubleDoubleToLongBits) -UNIMPLEMENTED_INTRINSIC(FloatIsNaN) -UNIMPLEMENTED_INTRINSIC(DoubleIsNaN) -UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft) -UNIMPLEMENTED_INTRINSIC(LongRotateLeft) -UNIMPLEMENTED_INTRINSIC(IntegerRotateRight) -UNIMPLEMENTED_INTRINSIC(LongRotateRight) -UNIMPLEMENTED_INTRINSIC(IntegerCompare) -UNIMPLEMENTED_INTRINSIC(LongCompare) -UNIMPLEMENTED_INTRINSIC(IntegerSignum) -UNIMPLEMENTED_INTRINSIC(LongSignum) +UNIMPLEMENTED_INTRINSIC(ARM64, SystemArrayCopy) +UNIMPLEMENTED_INTRINSIC(ARM64, ReferenceGetReferent) +UNIMPLEMENTED_INTRINSIC(ARM64, FloatIsInfinite) +UNIMPLEMENTED_INTRINSIC(ARM64, DoubleIsInfinite) +UNIMPLEMENTED_INTRINSIC(ARM64, IntegerHighestOneBit) +UNIMPLEMENTED_INTRINSIC(ARM64, LongHighestOneBit) +UNIMPLEMENTED_INTRINSIC(ARM64, IntegerLowestOneBit) +UNIMPLEMENTED_INTRINSIC(ARM64, LongLowestOneBit) -#undef UNIMPLEMENTED_INTRINSIC +UNREACHABLE_INTRINSICS(ARM64) #undef __ diff --git a/compiler/optimizing/intrinsics_list.h b/compiler/optimizing/intrinsics_list.h index e1aea924cf..b8933e1684 100644 --- a/compiler/optimizing/intrinsics_list.h +++ b/compiler/optimizing/intrinsics_list.h @@ -19,7 +19,7 @@ // All intrinsics supported by the optimizing compiler. Format is name, then whether it is expected // to be a HInvokeStaticOrDirect node (compared to HInvokeVirtual), then whether it requires an -// environment. +// environment, may have side effects, or may throw exceptions. #define INTRINSICS_LIST(V) \ V(DoubleDoubleToRawLongBits, kStatic, kNeedsEnvironmentOrCache, kNoSideEffects, kNoThrow) \ diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index a737d8100a..710df0a822 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -490,7 +490,6 @@ void IntrinsicCodeGeneratorMIPS::VisitLongNumberOfLeadingZeros(HInvoke* invoke) static void GenNumberOfTrailingZeroes(LocationSummary* locations, bool is64bit, bool isR6, - bool isR2OrNewer, MipsAssembler* assembler) { Register out = locations->Out().AsRegister<Register>(); Register in_lo; @@ -503,7 +502,7 @@ static void GenNumberOfTrailingZeroes(LocationSummary* locations, // If in_lo is zero then count the number of trailing zeroes in in_hi; // otherwise count the number of trailing zeroes in in_lo. - // AT = in_lo ? in_lo : in_hi; + // out = in_lo ? in_lo : in_hi; if (isR6) { __ Seleqz(out, in_hi, in_lo); __ Selnez(TMP, in_lo, in_lo); @@ -522,50 +521,26 @@ static void GenNumberOfTrailingZeroes(LocationSummary* locations, in_lo = in; } - // We don't have an instruction to count the number of trailing zeroes. - // Start by flipping the bits end-for-end so we can count the number of - // leading zeroes instead. - if (isR2OrNewer) { + if (isR6) { + // We don't have an instruction to count the number of trailing zeroes. + // Start by flipping the bits end-for-end so we can count the number of + // leading zeroes instead. __ Rotr(out, in, 16); __ Wsbh(out, out); - } else { - // MIPS32r1 - // __ Rotr(out, in, 16); - __ Sll(TMP, in, 16); - __ Srl(out, in, 16); - __ Or(out, out, TMP); - // __ Wsbh(out, out); - __ LoadConst32(AT, 0x00FF00FF); - __ And(TMP, out, AT); - __ Sll(TMP, TMP, 8); - __ Srl(out, out, 8); - __ And(out, out, AT); - __ Or(out, out, TMP); - } - - if (isR6) { __ Bitswap(out, out); __ ClzR6(out, out); } else { - __ LoadConst32(AT, 0x0F0F0F0F); - __ And(TMP, out, AT); - __ Sll(TMP, TMP, 4); - __ Srl(out, out, 4); - __ And(out, out, AT); - __ Or(out, TMP, out); - __ LoadConst32(AT, 0x33333333); - __ And(TMP, out, AT); - __ Sll(TMP, TMP, 2); - __ Srl(out, out, 2); - __ And(out, out, AT); - __ Or(out, TMP, out); - __ LoadConst32(AT, 0x55555555); - __ And(TMP, out, AT); - __ Sll(TMP, TMP, 1); - __ Srl(out, out, 1); - __ And(out, out, AT); - __ Or(out, TMP, out); + // Convert trailing zeroes to trailing ones, and bits to their left + // to zeroes. + __ Addiu(TMP, in, -1); + __ Xor(out, TMP, in); + __ And(out, out, TMP); + // Count number of leading zeroes. __ ClzR2(out, out); + // Subtract number of leading zeroes from 32 to get number of trailing ones. + // Remember that the trailing ones were formerly trailing zeroes. + __ LoadConst32(TMP, 32); + __ Subu(out, TMP, out); } if (is64bit) { @@ -587,11 +562,7 @@ void IntrinsicLocationsBuilderMIPS::VisitIntegerNumberOfTrailingZeros(HInvoke* i } void IntrinsicCodeGeneratorMIPS::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { - GenNumberOfTrailingZeroes(invoke->GetLocations(), - /* is64bit */ false, - IsR6(), - IsR2OrNewer(), - GetAssembler()); + GenNumberOfTrailingZeroes(invoke->GetLocations(), /* is64bit */ false, IsR6(), GetAssembler()); } // int java.lang.Long.numberOfTrailingZeros(long i) @@ -600,207 +571,7 @@ void IntrinsicLocationsBuilderMIPS::VisitLongNumberOfTrailingZeros(HInvoke* invo } void IntrinsicCodeGeneratorMIPS::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { - GenNumberOfTrailingZeroes(invoke->GetLocations(), - /* is64bit */ true, - IsR6(), - IsR2OrNewer(), - GetAssembler()); -} - -enum RotationDirection { - kRotateRight, - kRotateLeft, -}; - -static void GenRotate(HInvoke* invoke, - Primitive::Type type, - bool isR2OrNewer, - RotationDirection direction, - MipsAssembler* assembler) { - DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); - - LocationSummary* locations = invoke->GetLocations(); - if (invoke->InputAt(1)->IsIntConstant()) { - int32_t shift = static_cast<int32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()); - if (type == Primitive::kPrimInt) { - Register in = locations->InAt(0).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - shift &= 0x1f; - if (direction == kRotateLeft) { - shift = (32 - shift) & 0x1F; - } - - if (isR2OrNewer) { - if ((shift != 0) || (out != in)) { - __ Rotr(out, in, shift); - } - } else { - if (shift == 0) { - if (out != in) { - __ Move(out, in); - } - } else { - __ Srl(AT, in, shift); - __ Sll(out, in, 32 - shift); - __ Or(out, out, AT); - } - } - } else { // Primitive::kPrimLong - Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register out_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); - - shift &= 0x3f; - if (direction == kRotateLeft) { - shift = (64 - shift) & 0x3F; - } - - if (shift == 0) { - __ Move(out_lo, in_lo); - __ Move(out_hi, in_hi); - } else if (shift == 32) { - __ Move(out_lo, in_hi); - __ Move(out_hi, in_lo); - } else if (shift < 32) { - __ Srl(AT, in_lo, shift); - __ Sll(out_lo, in_hi, 32 - shift); - __ Or(out_lo, out_lo, AT); - __ Srl(AT, in_hi, shift); - __ Sll(out_hi, in_lo, 32 - shift); - __ Or(out_hi, out_hi, AT); - } else { - __ Sll(AT, in_lo, 64 - shift); - __ Srl(out_lo, in_hi, shift - 32); - __ Or(out_lo, out_lo, AT); - __ Sll(AT, in_hi, 64 - shift); - __ Srl(out_hi, in_lo, shift - 32); - __ Or(out_hi, out_hi, AT); - } - } - } else { // !invoke->InputAt(1)->IsIntConstant() - Register shamt = locations->InAt(1).AsRegister<Register>(); - if (type == Primitive::kPrimInt) { - Register in = locations->InAt(0).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - if (isR2OrNewer) { - if (direction == kRotateRight) { - __ Rotrv(out, in, shamt); - } else { - // negu tmp, shamt - __ Subu(TMP, ZERO, shamt); - __ Rotrv(out, in, TMP); - } - } else { - if (direction == kRotateRight) { - __ Srlv(AT, in, shamt); - __ Subu(TMP, ZERO, shamt); - __ Sllv(out, in, TMP); - __ Or(out, out, AT); - } else { - __ Sllv(AT, in, shamt); - __ Subu(TMP, ZERO, shamt); - __ Srlv(out, in, TMP); - __ Or(out, out, AT); - } - } - } else { // Primitive::kPrimLong - Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register out_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); - - MipsLabel done; - - if (direction == kRotateRight) { - __ Nor(TMP, ZERO, shamt); - __ Srlv(AT, in_lo, shamt); - __ Sll(out_lo, in_hi, 1); - __ Sllv(out_lo, out_lo, TMP); - __ Or(out_lo, out_lo, AT); - __ Srlv(AT, in_hi, shamt); - __ Sll(out_hi, in_lo, 1); - __ Sllv(out_hi, out_hi, TMP); - __ Or(out_hi, out_hi, AT); - } else { - __ Nor(TMP, ZERO, shamt); - __ Sllv(AT, in_lo, shamt); - __ Srl(out_lo, in_hi, 1); - __ Srlv(out_lo, out_lo, TMP); - __ Or(out_lo, out_lo, AT); - __ Sllv(AT, in_hi, shamt); - __ Srl(out_hi, in_lo, 1); - __ Srlv(out_hi, out_hi, TMP); - __ Or(out_hi, out_hi, AT); - } - - __ Andi(TMP, shamt, 32); - __ Beqz(TMP, &done); - __ Move(TMP, out_hi); - __ Move(out_hi, out_lo); - __ Move(out_lo, TMP); - - __ Bind(&done); - } - } -} - -// int java.lang.Integer.rotateRight(int i, int distance) -void IntrinsicLocationsBuilderMIPS::VisitIntegerRotateRight(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS::VisitIntegerRotateRight(HInvoke* invoke) { - GenRotate(invoke, Primitive::kPrimInt, IsR2OrNewer(), kRotateRight, GetAssembler()); -} - -// long java.lang.Long.rotateRight(long i, int distance) -void IntrinsicLocationsBuilderMIPS::VisitLongRotateRight(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS::VisitLongRotateRight(HInvoke* invoke) { - GenRotate(invoke, Primitive::kPrimLong, IsR2OrNewer(), kRotateRight, GetAssembler()); -} - -// int java.lang.Integer.rotateLeft(int i, int distance) -void IntrinsicLocationsBuilderMIPS::VisitIntegerRotateLeft(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS::VisitIntegerRotateLeft(HInvoke* invoke) { - GenRotate(invoke, Primitive::kPrimInt, IsR2OrNewer(), kRotateLeft, GetAssembler()); -} - -// long java.lang.Long.rotateLeft(long i, int distance) -void IntrinsicLocationsBuilderMIPS::VisitLongRotateLeft(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS::VisitLongRotateLeft(HInvoke* invoke) { - GenRotate(invoke, Primitive::kPrimLong, IsR2OrNewer(), kRotateLeft, GetAssembler()); + GenNumberOfTrailingZeroes(invoke->GetLocations(), /* is64bit */ true, IsR6(), GetAssembler()); } // int java.lang.Integer.reverse(int) @@ -839,6 +610,142 @@ static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); } +static void GenBitCount(LocationSummary* locations, + Primitive::Type type, + bool isR6, + MipsAssembler* assembler) { + DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); + + Register out = locations->Out().AsRegister<Register>(); + + // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + // + // A generalization of the best bit counting method to integers of + // bit-widths up to 128 (parameterized by type T) is this: + // + // v = v - ((v >> 1) & (T)~(T)0/3); // temp + // v = (v & (T)~(T)0/15*3) + ((v >> 2) & (T)~(T)0/15*3); // temp + // v = (v + (v >> 4)) & (T)~(T)0/255*15; // temp + // c = (T)(v * ((T)~(T)0/255)) >> (sizeof(T) - 1) * BITS_PER_BYTE; // count + // + // For comparison, for 32-bit quantities, this algorithm can be executed + // using 20 MIPS instructions (the calls to LoadConst32() generate two + // machine instructions each for the values being used in this algorithm). + // A(n unrolled) loop-based algorithm required 25 instructions. + // + // For 64-bit quantities, this algorithm gets executed twice, (once + // for in_lo, and again for in_hi), but saves a few instructions + // because the mask values only have to be loaded once. Using this + // algorithm the count for a 64-bit operand can be performed in 33 + // instructions compared to a loop-based algorithm which required 47 + // instructions. + + if (type == Primitive::kPrimInt) { + Register in = locations->InAt(0).AsRegister<Register>(); + + __ Srl(TMP, in, 1); + __ LoadConst32(AT, 0x55555555); + __ And(TMP, TMP, AT); + __ Subu(TMP, in, TMP); + __ LoadConst32(AT, 0x33333333); + __ And(out, TMP, AT); + __ Srl(TMP, TMP, 2); + __ And(TMP, TMP, AT); + __ Addu(TMP, out, TMP); + __ Srl(out, TMP, 4); + __ Addu(out, out, TMP); + __ LoadConst32(AT, 0x0F0F0F0F); + __ And(out, out, AT); + __ LoadConst32(TMP, 0x01010101); + if (isR6) { + __ MulR6(out, out, TMP); + } else { + __ MulR2(out, out, TMP); + } + __ Srl(out, out, 24); + } else if (type == Primitive::kPrimLong) { + Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register tmp_hi = locations->GetTemp(0).AsRegister<Register>(); + Register out_hi = locations->GetTemp(1).AsRegister<Register>(); + Register tmp_lo = TMP; + Register out_lo = out; + + __ Srl(tmp_lo, in_lo, 1); + __ Srl(tmp_hi, in_hi, 1); + + __ LoadConst32(AT, 0x55555555); + + __ And(tmp_lo, tmp_lo, AT); + __ Subu(tmp_lo, in_lo, tmp_lo); + + __ And(tmp_hi, tmp_hi, AT); + __ Subu(tmp_hi, in_hi, tmp_hi); + + __ LoadConst32(AT, 0x33333333); + + __ And(out_lo, tmp_lo, AT); + __ Srl(tmp_lo, tmp_lo, 2); + __ And(tmp_lo, tmp_lo, AT); + __ Addu(tmp_lo, out_lo, tmp_lo); + __ Srl(out_lo, tmp_lo, 4); + __ Addu(out_lo, out_lo, tmp_lo); + + __ And(out_hi, tmp_hi, AT); + __ Srl(tmp_hi, tmp_hi, 2); + __ And(tmp_hi, tmp_hi, AT); + __ Addu(tmp_hi, out_hi, tmp_hi); + __ Srl(out_hi, tmp_hi, 4); + __ Addu(out_hi, out_hi, tmp_hi); + + __ LoadConst32(AT, 0x0F0F0F0F); + + __ And(out_lo, out_lo, AT); + __ And(out_hi, out_hi, AT); + + __ LoadConst32(AT, 0x01010101); + + if (isR6) { + __ MulR6(out_lo, out_lo, AT); + + __ MulR6(out_hi, out_hi, AT); + } else { + __ MulR2(out_lo, out_lo, AT); + + __ MulR2(out_hi, out_hi, AT); + } + + __ Srl(out_lo, out_lo, 24); + __ Srl(out_hi, out_hi, 24); + + __ Addu(out, out_hi, out_lo); + } +} + +// int java.lang.Integer.bitCount(int) +void IntrinsicLocationsBuilderMIPS::VisitIntegerBitCount(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorMIPS::VisitIntegerBitCount(HInvoke* invoke) { + GenBitCount(invoke->GetLocations(), Primitive::kPrimInt, IsR6(), GetAssembler()); +} + +// int java.lang.Long.bitCount(int) +void IntrinsicLocationsBuilderMIPS::VisitLongBitCount(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); +} + +void IntrinsicCodeGeneratorMIPS::VisitLongBitCount(HInvoke* invoke) { + GenBitCount(invoke->GetLocations(), Primitive::kPrimLong, IsR6(), GetAssembler()); +} + static void MathAbsFP(LocationSummary* locations, bool is64bit, MipsAssembler* assembler) { FRegister in = locations->InAt(0).AsFpuRegister<FRegister>(); FRegister out = locations->Out().AsFpuRegister<FRegister>(); @@ -1698,88 +1605,234 @@ void IntrinsicCodeGeneratorMIPS::VisitStringEquals(HInvoke* invoke) { __ Bind(&end); } +static void GenIsInfinite(LocationSummary* locations, + const Primitive::Type type, + const bool isR6, + MipsAssembler* assembler) { + FRegister in = locations->InAt(0).AsFpuRegister<FRegister>(); + Register out = locations->Out().AsRegister<Register>(); + + DCHECK(type == Primitive::kPrimFloat || type == Primitive::kPrimDouble); + + if (isR6) { + if (type == Primitive::kPrimDouble) { + __ ClassD(FTMP, in); + } else { + __ ClassS(FTMP, in); + } + __ Mfc1(out, FTMP); + __ Andi(out, out, kPositiveInfinity | kNegativeInfinity); + __ Sltu(out, ZERO, out); + } else { + // If one, or more, of the exponent bits is zero, then the number can't be infinite. + if (type == Primitive::kPrimDouble) { + __ MoveFromFpuHigh(TMP, in); + __ LoadConst32(AT, 0x7FF00000); + } else { + __ Mfc1(TMP, in); + __ LoadConst32(AT, 0x7F800000); + } + __ Xor(TMP, TMP, AT); + + __ Sll(TMP, TMP, 1); + + if (type == Primitive::kPrimDouble) { + __ Mfc1(AT, in); + __ Or(TMP, TMP, AT); + } + // If any of the significand bits are one, then the number is not infinite. + __ Sltiu(out, TMP, 1); + } +} + +// boolean java.lang.Float.isInfinite(float) +void IntrinsicLocationsBuilderMIPS::VisitFloatIsInfinite(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorMIPS::VisitFloatIsInfinite(HInvoke* invoke) { + GenIsInfinite(invoke->GetLocations(), Primitive::kPrimFloat, IsR6(), GetAssembler()); +} + +// boolean java.lang.Double.isInfinite(double) +void IntrinsicLocationsBuilderMIPS::VisitDoubleIsInfinite(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorMIPS::VisitDoubleIsInfinite(HInvoke* invoke) { + GenIsInfinite(invoke->GetLocations(), Primitive::kPrimDouble, IsR6(), GetAssembler()); +} + +static void GenHighestOneBit(LocationSummary* locations, + const Primitive::Type type, + bool isR6, + MipsAssembler* assembler) { + DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); + + if (type == Primitive::kPrimLong) { + Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register out_lo = locations->Out().AsRegisterPairLow<Register>(); + Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); + + if (isR6) { + __ ClzR6(TMP, in_hi); + } else { + __ ClzR2(TMP, in_hi); + } + __ LoadConst32(AT, 0x80000000); + __ Srlv(out_hi, AT, TMP); + __ And(out_hi, out_hi, in_hi); + if (isR6) { + __ ClzR6(TMP, in_lo); + } else { + __ ClzR2(TMP, in_lo); + } + __ Srlv(out_lo, AT, TMP); + __ And(out_lo, out_lo, in_lo); + if (isR6) { + __ Seleqz(out_lo, out_lo, out_hi); + } else { + __ Movn(out_lo, ZERO, out_hi); + } + } else { + Register in = locations->InAt(0).AsRegister<Register>(); + Register out = locations->Out().AsRegister<Register>(); + + if (isR6) { + __ ClzR6(TMP, in); + } else { + __ ClzR2(TMP, in); + } + __ LoadConst32(AT, 0x80000000); + __ Srlv(AT, AT, TMP); // Srlv shifts in the range of [0;31] bits (lower 5 bits of arg). + __ And(out, AT, in); // So this is required for 0 (=shift by 32). + } +} + +// int java.lang.Integer.highestOneBit(int) +void IntrinsicLocationsBuilderMIPS::VisitIntegerHighestOneBit(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorMIPS::VisitIntegerHighestOneBit(HInvoke* invoke) { + GenHighestOneBit(invoke->GetLocations(), Primitive::kPrimInt, IsR6(), GetAssembler()); +} + +// long java.lang.Long.highestOneBit(long) +void IntrinsicLocationsBuilderMIPS::VisitLongHighestOneBit(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke, Location::kOutputOverlap); +} + +void IntrinsicCodeGeneratorMIPS::VisitLongHighestOneBit(HInvoke* invoke) { + GenHighestOneBit(invoke->GetLocations(), Primitive::kPrimLong, IsR6(), GetAssembler()); +} + +static void GenLowestOneBit(LocationSummary* locations, + const Primitive::Type type, + bool isR6, + MipsAssembler* assembler) { + DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); + + if (type == Primitive::kPrimLong) { + Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register out_lo = locations->Out().AsRegisterPairLow<Register>(); + Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); + + __ Subu(TMP, ZERO, in_lo); + __ And(out_lo, TMP, in_lo); + __ Subu(TMP, ZERO, in_hi); + __ And(out_hi, TMP, in_hi); + if (isR6) { + __ Seleqz(out_hi, out_hi, out_lo); + } else { + __ Movn(out_hi, ZERO, out_lo); + } + } else { + Register in = locations->InAt(0).AsRegister<Register>(); + Register out = locations->Out().AsRegister<Register>(); + + __ Subu(TMP, ZERO, in); + __ And(out, TMP, in); + } +} + +// int java.lang.Integer.lowestOneBit(int) +void IntrinsicLocationsBuilderMIPS::VisitIntegerLowestOneBit(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorMIPS::VisitIntegerLowestOneBit(HInvoke* invoke) { + GenLowestOneBit(invoke->GetLocations(), Primitive::kPrimInt, IsR6(), GetAssembler()); +} + +// long java.lang.Long.lowestOneBit(long) +void IntrinsicLocationsBuilderMIPS::VisitLongLowestOneBit(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorMIPS::VisitLongLowestOneBit(HInvoke* invoke) { + GenLowestOneBit(invoke->GetLocations(), Primitive::kPrimLong, IsR6(), GetAssembler()); +} + // Unimplemented intrinsics. -#define UNIMPLEMENTED_INTRINSIC(Name) \ -void IntrinsicLocationsBuilderMIPS::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} \ -void IntrinsicCodeGeneratorMIPS::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} - -UNIMPLEMENTED_INTRINSIC(IntegerBitCount) -UNIMPLEMENTED_INTRINSIC(LongBitCount) - -UNIMPLEMENTED_INTRINSIC(MathCeil) -UNIMPLEMENTED_INTRINSIC(MathFloor) -UNIMPLEMENTED_INTRINSIC(MathRint) -UNIMPLEMENTED_INTRINSIC(MathRoundDouble) -UNIMPLEMENTED_INTRINSIC(MathRoundFloat) -UNIMPLEMENTED_INTRINSIC(ThreadCurrentThread) -UNIMPLEMENTED_INTRINSIC(UnsafeGet) -UNIMPLEMENTED_INTRINSIC(UnsafeGetVolatile) -UNIMPLEMENTED_INTRINSIC(UnsafeGetLong) -UNIMPLEMENTED_INTRINSIC(UnsafeGetLongVolatile) -UNIMPLEMENTED_INTRINSIC(UnsafeGetObject) -UNIMPLEMENTED_INTRINSIC(UnsafeGetObjectVolatile) -UNIMPLEMENTED_INTRINSIC(UnsafePut) -UNIMPLEMENTED_INTRINSIC(UnsafePutOrdered) -UNIMPLEMENTED_INTRINSIC(UnsafePutVolatile) -UNIMPLEMENTED_INTRINSIC(UnsafePutObject) -UNIMPLEMENTED_INTRINSIC(UnsafePutObjectOrdered) -UNIMPLEMENTED_INTRINSIC(UnsafePutObjectVolatile) -UNIMPLEMENTED_INTRINSIC(UnsafePutLong) -UNIMPLEMENTED_INTRINSIC(UnsafePutLongOrdered) -UNIMPLEMENTED_INTRINSIC(UnsafePutLongVolatile) -UNIMPLEMENTED_INTRINSIC(UnsafeCASInt) -UNIMPLEMENTED_INTRINSIC(UnsafeCASLong) -UNIMPLEMENTED_INTRINSIC(UnsafeCASObject) -UNIMPLEMENTED_INTRINSIC(StringCompareTo) -UNIMPLEMENTED_INTRINSIC(StringIndexOf) -UNIMPLEMENTED_INTRINSIC(StringIndexOfAfter) -UNIMPLEMENTED_INTRINSIC(StringNewStringFromBytes) -UNIMPLEMENTED_INTRINSIC(StringNewStringFromChars) -UNIMPLEMENTED_INTRINSIC(StringNewStringFromString) - -UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) -UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck) -UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar) -UNIMPLEMENTED_INTRINSIC(SystemArrayCopy) - -UNIMPLEMENTED_INTRINSIC(MathCos) -UNIMPLEMENTED_INTRINSIC(MathSin) -UNIMPLEMENTED_INTRINSIC(MathAcos) -UNIMPLEMENTED_INTRINSIC(MathAsin) -UNIMPLEMENTED_INTRINSIC(MathAtan) -UNIMPLEMENTED_INTRINSIC(MathAtan2) -UNIMPLEMENTED_INTRINSIC(MathCbrt) -UNIMPLEMENTED_INTRINSIC(MathCosh) -UNIMPLEMENTED_INTRINSIC(MathExp) -UNIMPLEMENTED_INTRINSIC(MathExpm1) -UNIMPLEMENTED_INTRINSIC(MathHypot) -UNIMPLEMENTED_INTRINSIC(MathLog) -UNIMPLEMENTED_INTRINSIC(MathLog10) -UNIMPLEMENTED_INTRINSIC(MathNextAfter) -UNIMPLEMENTED_INTRINSIC(MathSinh) -UNIMPLEMENTED_INTRINSIC(MathTan) -UNIMPLEMENTED_INTRINSIC(MathTanh) - -UNIMPLEMENTED_INTRINSIC(FloatIsInfinite) -UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite) - -UNIMPLEMENTED_INTRINSIC(IntegerHighestOneBit) -UNIMPLEMENTED_INTRINSIC(LongHighestOneBit) -UNIMPLEMENTED_INTRINSIC(IntegerLowestOneBit) -UNIMPLEMENTED_INTRINSIC(LongLowestOneBit) - -// Handled as HIR instructions. -UNIMPLEMENTED_INTRINSIC(FloatFloatToIntBits) -UNIMPLEMENTED_INTRINSIC(DoubleDoubleToLongBits) -UNIMPLEMENTED_INTRINSIC(FloatIsNaN) -UNIMPLEMENTED_INTRINSIC(DoubleIsNaN) -UNIMPLEMENTED_INTRINSIC(IntegerCompare) -UNIMPLEMENTED_INTRINSIC(LongCompare) -UNIMPLEMENTED_INTRINSIC(IntegerSignum) -UNIMPLEMENTED_INTRINSIC(LongSignum) +UNIMPLEMENTED_INTRINSIC(MIPS, MathCeil) +UNIMPLEMENTED_INTRINSIC(MIPS, MathFloor) +UNIMPLEMENTED_INTRINSIC(MIPS, MathRint) +UNIMPLEMENTED_INTRINSIC(MIPS, MathRoundDouble) +UNIMPLEMENTED_INTRINSIC(MIPS, MathRoundFloat) +UNIMPLEMENTED_INTRINSIC(MIPS, ThreadCurrentThread) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeGet) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeGetVolatile) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeGetLong) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeGetLongVolatile) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeGetObject) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeGetObjectVolatile) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePut) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutOrdered) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutVolatile) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutObject) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutObjectOrdered) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutObjectVolatile) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutLong) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutLongOrdered) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutLongVolatile) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeCASInt) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeCASLong) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeCASObject) +UNIMPLEMENTED_INTRINSIC(MIPS, StringCompareTo) +UNIMPLEMENTED_INTRINSIC(MIPS, StringIndexOf) +UNIMPLEMENTED_INTRINSIC(MIPS, StringIndexOfAfter) +UNIMPLEMENTED_INTRINSIC(MIPS, StringNewStringFromBytes) +UNIMPLEMENTED_INTRINSIC(MIPS, StringNewStringFromChars) +UNIMPLEMENTED_INTRINSIC(MIPS, StringNewStringFromString) + +UNIMPLEMENTED_INTRINSIC(MIPS, ReferenceGetReferent) +UNIMPLEMENTED_INTRINSIC(MIPS, StringGetCharsNoCheck) +UNIMPLEMENTED_INTRINSIC(MIPS, SystemArrayCopyChar) +UNIMPLEMENTED_INTRINSIC(MIPS, SystemArrayCopy) + +UNIMPLEMENTED_INTRINSIC(MIPS, MathCos) +UNIMPLEMENTED_INTRINSIC(MIPS, MathSin) +UNIMPLEMENTED_INTRINSIC(MIPS, MathAcos) +UNIMPLEMENTED_INTRINSIC(MIPS, MathAsin) +UNIMPLEMENTED_INTRINSIC(MIPS, MathAtan) +UNIMPLEMENTED_INTRINSIC(MIPS, MathAtan2) +UNIMPLEMENTED_INTRINSIC(MIPS, MathCbrt) +UNIMPLEMENTED_INTRINSIC(MIPS, MathCosh) +UNIMPLEMENTED_INTRINSIC(MIPS, MathExp) +UNIMPLEMENTED_INTRINSIC(MIPS, MathExpm1) +UNIMPLEMENTED_INTRINSIC(MIPS, MathHypot) +UNIMPLEMENTED_INTRINSIC(MIPS, MathLog) +UNIMPLEMENTED_INTRINSIC(MIPS, MathLog10) +UNIMPLEMENTED_INTRINSIC(MIPS, MathNextAfter) +UNIMPLEMENTED_INTRINSIC(MIPS, MathSinh) +UNIMPLEMENTED_INTRINSIC(MIPS, MathTan) +UNIMPLEMENTED_INTRINSIC(MIPS, MathTanh) + +UNREACHABLE_INTRINSICS(MIPS) #undef UNIMPLEMENTED_INTRINSIC diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index ca2652b74a..45611f0ac7 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -340,130 +340,6 @@ void IntrinsicCodeGeneratorMIPS64::VisitLongNumberOfTrailingZeros(HInvoke* invok GenNumberOfTrailingZeroes(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); } -static void GenRotateRight(HInvoke* invoke, - Primitive::Type type, - Mips64Assembler* assembler) { - DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); - - LocationSummary* locations = invoke->GetLocations(); - GpuRegister in = locations->InAt(0).AsRegister<GpuRegister>(); - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - - if (invoke->InputAt(1)->IsIntConstant()) { - uint32_t shift = static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()); - if (type == Primitive::kPrimInt) { - shift &= 0x1f; - __ Rotr(out, in, shift); - } else { - shift &= 0x3f; - if (shift < 32) { - __ Drotr(out, in, shift); - } else { - shift &= 0x1f; - __ Drotr32(out, in, shift); - } - } - } else { - GpuRegister shamt = locations->InAt(1).AsRegister<GpuRegister>(); - if (type == Primitive::kPrimInt) { - __ Rotrv(out, in, shamt); - } else { - __ Drotrv(out, in, shamt); - } - } -} - -// int java.lang.Integer.rotateRight(int i, int distance) -void IntrinsicLocationsBuilderMIPS64::VisitIntegerRotateRight(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS64::VisitIntegerRotateRight(HInvoke* invoke) { - GenRotateRight(invoke, Primitive::kPrimInt, GetAssembler()); -} - -// long java.lang.Long.rotateRight(long i, int distance) -void IntrinsicLocationsBuilderMIPS64::VisitLongRotateRight(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS64::VisitLongRotateRight(HInvoke* invoke) { - GenRotateRight(invoke, Primitive::kPrimLong, GetAssembler()); -} - -static void GenRotateLeft(HInvoke* invoke, - Primitive::Type type, - Mips64Assembler* assembler) { - DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); - - LocationSummary* locations = invoke->GetLocations(); - GpuRegister in = locations->InAt(0).AsRegister<GpuRegister>(); - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - - if (invoke->InputAt(1)->IsIntConstant()) { - int32_t shift = -static_cast<int32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()); - if (type == Primitive::kPrimInt) { - shift &= 0x1f; - __ Rotr(out, in, shift); - } else { - shift &= 0x3f; - if (shift < 32) { - __ Drotr(out, in, shift); - } else { - shift &= 0x1f; - __ Drotr32(out, in, shift); - } - } - } else { - GpuRegister shamt = locations->InAt(1).AsRegister<GpuRegister>(); - if (type == Primitive::kPrimInt) { - __ Subu(TMP, ZERO, shamt); - __ Rotrv(out, in, TMP); - } else { - __ Dsubu(TMP, ZERO, shamt); - __ Drotrv(out, in, TMP); - } - } -} - -// int java.lang.Integer.rotateLeft(int i, int distance) -void IntrinsicLocationsBuilderMIPS64::VisitIntegerRotateLeft(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS64::VisitIntegerRotateLeft(HInvoke* invoke) { - GenRotateLeft(invoke, Primitive::kPrimInt, GetAssembler()); -} - -// long java.lang.Long.rotateLeft(long i, int distance) -void IntrinsicLocationsBuilderMIPS64::VisitLongRotateLeft(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorMIPS64::VisitLongRotateLeft(HInvoke* invoke) { - GenRotateLeft(invoke, Primitive::kPrimLong, GetAssembler()); -} - static void GenReverse(LocationSummary* locations, Primitive::Type type, Mips64Assembler* assembler) { @@ -1636,6 +1512,7 @@ static void GenerateStringIndexOf(HInvoke* invoke, TMP, TR, QUICK_ENTRYPOINT_OFFSET(kMips64DoublewordSize, pIndexOf).Int32Value()); + CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>(); __ Jalr(TMP); __ Nop(); @@ -1685,7 +1562,7 @@ void IntrinsicCodeGeneratorMIPS64::VisitStringIndexOfAfter(HInvoke* invoke) { invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); } -// java.lang.String.String(byte[] bytes) +// java.lang.StringFactory.newStringFromBytes(byte[] data, int high, int offset, int byteCount) void IntrinsicLocationsBuilderMIPS64::VisitStringNewStringFromBytes(HInvoke* invoke) { LocationSummary* locations = new (arena_) LocationSummary(invoke, LocationSummary::kCall, @@ -1719,7 +1596,7 @@ void IntrinsicCodeGeneratorMIPS64::VisitStringNewStringFromBytes(HInvoke* invoke __ Bind(slow_path->GetExitLabel()); } -// java.lang.String.String(char[] value) +// java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) void IntrinsicLocationsBuilderMIPS64::VisitStringNewStringFromChars(HInvoke* invoke) { LocationSummary* locations = new (arena_) LocationSummary(invoke, LocationSummary::kCall, @@ -1735,6 +1612,12 @@ void IntrinsicLocationsBuilderMIPS64::VisitStringNewStringFromChars(HInvoke* inv void IntrinsicCodeGeneratorMIPS64::VisitStringNewStringFromChars(HInvoke* invoke) { Mips64Assembler* assembler = GetAssembler(); + // No need to emit code checking whether `locations->InAt(2)` is a null + // pointer, as callers of the native method + // + // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) + // + // all include a null check on `data` before calling that method. __ LoadFromOffset(kLoadDoubleword, TMP, TR, @@ -1778,62 +1661,44 @@ void IntrinsicCodeGeneratorMIPS64::VisitStringNewStringFromString(HInvoke* invok __ Bind(slow_path->GetExitLabel()); } -// Unimplemented intrinsics. - -#define UNIMPLEMENTED_INTRINSIC(Name) \ -void IntrinsicLocationsBuilderMIPS64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} \ -void IntrinsicCodeGeneratorMIPS64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} - -UNIMPLEMENTED_INTRINSIC(IntegerBitCount) -UNIMPLEMENTED_INTRINSIC(LongBitCount) - -UNIMPLEMENTED_INTRINSIC(MathRoundDouble) -UNIMPLEMENTED_INTRINSIC(MathRoundFloat) - -UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) -UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck) -UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar) -UNIMPLEMENTED_INTRINSIC(SystemArrayCopy) - -UNIMPLEMENTED_INTRINSIC(MathCos) -UNIMPLEMENTED_INTRINSIC(MathSin) -UNIMPLEMENTED_INTRINSIC(MathAcos) -UNIMPLEMENTED_INTRINSIC(MathAsin) -UNIMPLEMENTED_INTRINSIC(MathAtan) -UNIMPLEMENTED_INTRINSIC(MathAtan2) -UNIMPLEMENTED_INTRINSIC(MathCbrt) -UNIMPLEMENTED_INTRINSIC(MathCosh) -UNIMPLEMENTED_INTRINSIC(MathExp) -UNIMPLEMENTED_INTRINSIC(MathExpm1) -UNIMPLEMENTED_INTRINSIC(MathHypot) -UNIMPLEMENTED_INTRINSIC(MathLog) -UNIMPLEMENTED_INTRINSIC(MathLog10) -UNIMPLEMENTED_INTRINSIC(MathNextAfter) -UNIMPLEMENTED_INTRINSIC(MathSinh) -UNIMPLEMENTED_INTRINSIC(MathTan) -UNIMPLEMENTED_INTRINSIC(MathTanh) - -UNIMPLEMENTED_INTRINSIC(FloatIsInfinite) -UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite) - -UNIMPLEMENTED_INTRINSIC(IntegerHighestOneBit) -UNIMPLEMENTED_INTRINSIC(LongHighestOneBit) -UNIMPLEMENTED_INTRINSIC(IntegerLowestOneBit) -UNIMPLEMENTED_INTRINSIC(LongLowestOneBit) - -// Handled as HIR instructions. -UNIMPLEMENTED_INTRINSIC(FloatFloatToIntBits) -UNIMPLEMENTED_INTRINSIC(DoubleDoubleToLongBits) -UNIMPLEMENTED_INTRINSIC(FloatIsNaN) -UNIMPLEMENTED_INTRINSIC(DoubleIsNaN) -UNIMPLEMENTED_INTRINSIC(IntegerCompare) -UNIMPLEMENTED_INTRINSIC(LongCompare) -UNIMPLEMENTED_INTRINSIC(IntegerSignum) -UNIMPLEMENTED_INTRINSIC(LongSignum) - -#undef UNIMPLEMENTED_INTRINSIC +UNIMPLEMENTED_INTRINSIC(MIPS64, IntegerBitCount) +UNIMPLEMENTED_INTRINSIC(MIPS64, LongBitCount) + +UNIMPLEMENTED_INTRINSIC(MIPS64, MathRoundDouble) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathRoundFloat) + +UNIMPLEMENTED_INTRINSIC(MIPS64, ReferenceGetReferent) +UNIMPLEMENTED_INTRINSIC(MIPS64, StringGetCharsNoCheck) +UNIMPLEMENTED_INTRINSIC(MIPS64, SystemArrayCopyChar) +UNIMPLEMENTED_INTRINSIC(MIPS64, SystemArrayCopy) + +UNIMPLEMENTED_INTRINSIC(MIPS64, MathCos) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathSin) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathAcos) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathAsin) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathAtan) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathAtan2) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathCbrt) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathCosh) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathExp) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathExpm1) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathHypot) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathLog) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathLog10) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathNextAfter) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathSinh) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathTan) +UNIMPLEMENTED_INTRINSIC(MIPS64, MathTanh) + +UNIMPLEMENTED_INTRINSIC(MIPS64, FloatIsInfinite) +UNIMPLEMENTED_INTRINSIC(MIPS64, DoubleIsInfinite) + +UNIMPLEMENTED_INTRINSIC(MIPS64, IntegerHighestOneBit) +UNIMPLEMENTED_INTRINSIC(MIPS64, LongHighestOneBit) +UNIMPLEMENTED_INTRINSIC(MIPS64, IntegerLowestOneBit) +UNIMPLEMENTED_INTRINSIC(MIPS64, LongLowestOneBit) + +UNREACHABLE_INTRINSICS(MIPS64) #undef __ diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index 0df4553f56..9a2dc4182d 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -1564,6 +1564,12 @@ void IntrinsicLocationsBuilderX86::VisitStringNewStringFromChars(HInvoke* invoke void IntrinsicCodeGeneratorX86::VisitStringNewStringFromChars(HInvoke* invoke) { X86Assembler* assembler = GetAssembler(); + // No need to emit code checking whether `locations->InAt(2)` is a null + // pointer, as callers of the native method + // + // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) + // + // all include a null check on `data` before calling that method. __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAllocStringFromChars))); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } @@ -2621,41 +2627,17 @@ void IntrinsicCodeGeneratorX86::VisitLongNumberOfTrailingZeros(HInvoke* invoke) GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long */ true); } -// Unimplemented intrinsics. - -#define UNIMPLEMENTED_INTRINSIC(Name) \ -void IntrinsicLocationsBuilderX86::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} \ -void IntrinsicCodeGeneratorX86::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} - -UNIMPLEMENTED_INTRINSIC(MathRoundDouble) -UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) -UNIMPLEMENTED_INTRINSIC(SystemArrayCopy) - -UNIMPLEMENTED_INTRINSIC(FloatIsInfinite) -UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite) - -UNIMPLEMENTED_INTRINSIC(IntegerHighestOneBit) -UNIMPLEMENTED_INTRINSIC(LongHighestOneBit) -UNIMPLEMENTED_INTRINSIC(IntegerLowestOneBit) -UNIMPLEMENTED_INTRINSIC(LongLowestOneBit) - -// Handled as HIR instructions. -UNIMPLEMENTED_INTRINSIC(FloatFloatToIntBits) -UNIMPLEMENTED_INTRINSIC(DoubleDoubleToLongBits) -UNIMPLEMENTED_INTRINSIC(FloatIsNaN) -UNIMPLEMENTED_INTRINSIC(DoubleIsNaN) -UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft) -UNIMPLEMENTED_INTRINSIC(LongRotateLeft) -UNIMPLEMENTED_INTRINSIC(IntegerRotateRight) -UNIMPLEMENTED_INTRINSIC(LongRotateRight) -UNIMPLEMENTED_INTRINSIC(IntegerCompare) -UNIMPLEMENTED_INTRINSIC(LongCompare) -UNIMPLEMENTED_INTRINSIC(IntegerSignum) -UNIMPLEMENTED_INTRINSIC(LongSignum) +UNIMPLEMENTED_INTRINSIC(X86, MathRoundDouble) +UNIMPLEMENTED_INTRINSIC(X86, ReferenceGetReferent) +UNIMPLEMENTED_INTRINSIC(X86, SystemArrayCopy) +UNIMPLEMENTED_INTRINSIC(X86, FloatIsInfinite) +UNIMPLEMENTED_INTRINSIC(X86, DoubleIsInfinite) +UNIMPLEMENTED_INTRINSIC(X86, IntegerHighestOneBit) +UNIMPLEMENTED_INTRINSIC(X86, LongHighestOneBit) +UNIMPLEMENTED_INTRINSIC(X86, IntegerLowestOneBit) +UNIMPLEMENTED_INTRINSIC(X86, LongLowestOneBit) -#undef UNIMPLEMENTED_INTRINSIC +UNREACHABLE_INTRINSICS(X86) #undef __ diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index 2a9e684d11..75204b4b49 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -1659,6 +1659,12 @@ void IntrinsicLocationsBuilderX86_64::VisitStringNewStringFromChars(HInvoke* inv void IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromChars(HInvoke* invoke) { X86_64Assembler* assembler = GetAssembler(); + // No need to emit code checking whether `locations->InAt(2)` is a null + // pointer, as callers of the native method + // + // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) + // + // all include a null check on `data` before calling that method. __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocStringFromChars), /* no_rip */ true)); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); @@ -2705,34 +2711,11 @@ void IntrinsicCodeGeneratorX86_64::VisitLongNumberOfTrailingZeros(HInvoke* invok GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long */ true); } -// Unimplemented intrinsics. - -#define UNIMPLEMENTED_INTRINSIC(Name) \ -void IntrinsicLocationsBuilderX86_64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} \ -void IntrinsicCodeGeneratorX86_64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ -} - -UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) - -UNIMPLEMENTED_INTRINSIC(FloatIsInfinite) -UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite) - -// Handled as HIR instructions. -UNIMPLEMENTED_INTRINSIC(FloatFloatToIntBits) -UNIMPLEMENTED_INTRINSIC(DoubleDoubleToLongBits) -UNIMPLEMENTED_INTRINSIC(FloatIsNaN) -UNIMPLEMENTED_INTRINSIC(DoubleIsNaN) -UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft) -UNIMPLEMENTED_INTRINSIC(LongRotateLeft) -UNIMPLEMENTED_INTRINSIC(IntegerRotateRight) -UNIMPLEMENTED_INTRINSIC(LongRotateRight) -UNIMPLEMENTED_INTRINSIC(IntegerCompare) -UNIMPLEMENTED_INTRINSIC(LongCompare) -UNIMPLEMENTED_INTRINSIC(IntegerSignum) -UNIMPLEMENTED_INTRINSIC(LongSignum) +UNIMPLEMENTED_INTRINSIC(X86_64, ReferenceGetReferent) +UNIMPLEMENTED_INTRINSIC(X86_64, FloatIsInfinite) +UNIMPLEMENTED_INTRINSIC(X86_64, DoubleIsInfinite) -#undef UNIMPLEMENTED_INTRINSIC +UNREACHABLE_INTRINSICS(X86_64) #undef __ diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 27a5b97f5f..77ded29b49 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -127,6 +127,9 @@ void HGraph::RemoveDeadBlocks(const ArenaBitVector& visited) { // Remove the block from the list of blocks, so that further analyses // never see it. blocks_[i] = nullptr; + if (block->IsExitBlock()) { + SetExitBlock(nullptr); + } } } } @@ -1870,13 +1873,49 @@ void HGraph::DeleteDeadEmptyBlock(HBasicBlock* block) { DCHECK(block->GetPhis().IsEmpty()); if (block->IsExitBlock()) { - exit_block_ = nullptr; + SetExitBlock(nullptr); } RemoveElement(reverse_post_order_, block); blocks_[block->GetBlockId()] = nullptr; } +void HGraph::UpdateLoopAndTryInformationOfNewBlock(HBasicBlock* block, + HBasicBlock* reference, + bool replace_if_back_edge) { + if (block->IsLoopHeader()) { + // Clear the information of which blocks are contained in that loop. Since the + // information is stored as a bit vector based on block ids, we have to update + // it, as those block ids were specific to the callee graph and we are now adding + // these blocks to the caller graph. + block->GetLoopInformation()->ClearAllBlocks(); + } + + // If not already in a loop, update the loop information. + if (!block->IsInLoop()) { + block->SetLoopInformation(reference->GetLoopInformation()); + } + + // If the block is in a loop, update all its outward loops. + HLoopInformation* loop_info = block->GetLoopInformation(); + if (loop_info != nullptr) { + for (HLoopInformationOutwardIterator loop_it(*block); + !loop_it.Done(); + loop_it.Advance()) { + loop_it.Current()->Add(block); + } + if (replace_if_back_edge && loop_info->IsBackEdge(*reference)) { + loop_info->ReplaceBackEdge(reference, block); + } + } + + // Copy TryCatchInformation if `reference` is a try block, not if it is a catch block. + TryCatchInformation* try_catch_info = reference->IsTryBlock() + ? reference->GetTryCatchInformation() + : nullptr; + block->SetTryCatchInformation(try_catch_info); +} + HInstruction* HGraph::InlineInto(HGraph* outer_graph, HInvoke* invoke) { DCHECK(HasExitBlock()) << "Unimplemented scenario"; // Update the environments in this graph to have the invoke's environment @@ -1940,34 +1979,6 @@ HInstruction* HGraph::InlineInto(HGraph* outer_graph, HInvoke* invoke) { at->MergeWithInlined(first); exit_block_->ReplaceWith(to); - // Update all predecessors of the exit block (now the `to` block) - // to not `HReturn` but `HGoto` instead. - bool returns_void = to->GetPredecessors()[0]->GetLastInstruction()->IsReturnVoid(); - if (to->GetPredecessors().size() == 1) { - HBasicBlock* predecessor = to->GetPredecessors()[0]; - HInstruction* last = predecessor->GetLastInstruction(); - if (!returns_void) { - return_value = last->InputAt(0); - } - predecessor->AddInstruction(new (allocator) HGoto(last->GetDexPc())); - predecessor->RemoveInstruction(last); - } else { - if (!returns_void) { - // There will be multiple returns. - return_value = new (allocator) HPhi( - allocator, kNoRegNumber, 0, HPhi::ToPhiType(invoke->GetType()), to->GetDexPc()); - to->AddPhi(return_value->AsPhi()); - } - for (HBasicBlock* predecessor : to->GetPredecessors()) { - HInstruction* last = predecessor->GetLastInstruction(); - if (!returns_void) { - return_value->AsPhi()->AddInput(last->InputAt(0)); - } - predecessor->AddInstruction(new (allocator) HGoto(last->GetDexPc())); - predecessor->RemoveInstruction(last); - } - } - // Update the meta information surrounding blocks: // (1) the graph they are now in, // (2) the reverse post order of that graph, @@ -1991,10 +2002,6 @@ HInstruction* HGraph::InlineInto(HGraph* outer_graph, HInvoke* invoke) { size_t index_of_at = IndexOfElement(outer_graph->reverse_post_order_, at); MakeRoomFor(&outer_graph->reverse_post_order_, blocks_added, index_of_at); - HLoopInformation* loop_info = at->GetLoopInformation(); - // Copy TryCatchInformation if `at` is a try block, not if it is a catch block. - TryCatchInformation* try_catch_info = at->IsTryBlock() ? at->GetTryCatchInformation() : nullptr; - // Do a reverse post order of the blocks in the callee and do (1), (2), (3) // and (4) to the blocks that apply. for (HReversePostOrderIterator it(*this); !it.Done(); it.Advance()) { @@ -2005,23 +2012,7 @@ HInstruction* HGraph::InlineInto(HGraph* outer_graph, HInvoke* invoke) { current->SetGraph(outer_graph); outer_graph->AddBlock(current); outer_graph->reverse_post_order_[++index_of_at] = current; - if (!current->IsInLoop()) { - current->SetLoopInformation(loop_info); - } else if (current->IsLoopHeader()) { - // Clear the information of which blocks are contained in that loop. Since the - // information is stored as a bit vector based on block ids, we have to update - // it, as those block ids were specific to the callee graph and we are now adding - // these blocks to the caller graph. - current->GetLoopInformation()->ClearAllBlocks(); - } - if (current->IsInLoop()) { - for (HLoopInformationOutwardIterator loop_it(*current); - !loop_it.Done(); - loop_it.Advance()) { - loop_it.Current()->Add(current); - } - } - current->SetTryCatchInformation(try_catch_info); + UpdateLoopAndTryInformationOfNewBlock(current, at, /* replace_if_back_edge */ false); } } @@ -2029,26 +2020,40 @@ HInstruction* HGraph::InlineInto(HGraph* outer_graph, HInvoke* invoke) { to->SetGraph(outer_graph); outer_graph->AddBlock(to); outer_graph->reverse_post_order_[++index_of_at] = to; - if (loop_info != nullptr) { - if (!to->IsInLoop()) { - to->SetLoopInformation(loop_info); + // Only `to` can become a back edge, as the inlined blocks + // are predecessors of `to`. + UpdateLoopAndTryInformationOfNewBlock(to, at, /* replace_if_back_edge */ true); + + // Update all predecessors of the exit block (now the `to` block) + // to not `HReturn` but `HGoto` instead. + bool returns_void = to->GetPredecessors()[0]->GetLastInstruction()->IsReturnVoid(); + if (to->GetPredecessors().size() == 1) { + HBasicBlock* predecessor = to->GetPredecessors()[0]; + HInstruction* last = predecessor->GetLastInstruction(); + if (!returns_void) { + return_value = last->InputAt(0); } - for (HLoopInformationOutwardIterator loop_it(*at); !loop_it.Done(); loop_it.Advance()) { - loop_it.Current()->Add(to); + predecessor->AddInstruction(new (allocator) HGoto(last->GetDexPc())); + predecessor->RemoveInstruction(last); + } else { + if (!returns_void) { + // There will be multiple returns. + return_value = new (allocator) HPhi( + allocator, kNoRegNumber, 0, HPhi::ToPhiType(invoke->GetType()), to->GetDexPc()); + to->AddPhi(return_value->AsPhi()); } - if (loop_info->IsBackEdge(*at)) { - // Only `to` can become a back edge, as the inlined blocks - // are predecessors of `to`. - loop_info->ReplaceBackEdge(at, to); + for (HBasicBlock* predecessor : to->GetPredecessors()) { + HInstruction* last = predecessor->GetLastInstruction(); + if (!returns_void) { + DCHECK(last->IsReturn()); + return_value->AsPhi()->AddInput(last->InputAt(0)); + } + predecessor->AddInstruction(new (allocator) HGoto(last->GetDexPc())); + predecessor->RemoveInstruction(last); } } - to->SetTryCatchInformation(try_catch_info); } - // Update the next instruction id of the outer graph, so that instructions - // added later get bigger ids than those in the inner graph. - outer_graph->SetCurrentInstructionId(GetNextInstructionId()); - // Walk over the entry block and: // - Move constants from the entry block to the outer_graph's entry block, // - Replace HParameterValue instructions with their real value. @@ -2157,32 +2162,17 @@ void HGraph::TransformLoopHeaderForBCE(HBasicBlock* header) { reverse_post_order_[index_of_header++] = false_block; reverse_post_order_[index_of_header++] = new_pre_header; - // Fix loop information. - HLoopInformation* loop_info = old_pre_header->GetLoopInformation(); - if (loop_info != nullptr) { - if_block->SetLoopInformation(loop_info); - true_block->SetLoopInformation(loop_info); - false_block->SetLoopInformation(loop_info); - new_pre_header->SetLoopInformation(loop_info); - // Add blocks to all enveloping loops. - for (HLoopInformationOutwardIterator loop_it(*old_pre_header); - !loop_it.Done(); - loop_it.Advance()) { - loop_it.Current()->Add(if_block); - loop_it.Current()->Add(true_block); - loop_it.Current()->Add(false_block); - loop_it.Current()->Add(new_pre_header); - } - } - - // Fix try/catch information. - TryCatchInformation* try_catch_info = old_pre_header->IsTryBlock() - ? old_pre_header->GetTryCatchInformation() - : nullptr; - if_block->SetTryCatchInformation(try_catch_info); - true_block->SetTryCatchInformation(try_catch_info); - false_block->SetTryCatchInformation(try_catch_info); - new_pre_header->SetTryCatchInformation(try_catch_info); + // The pre_header can never be a back edge of a loop. + DCHECK((old_pre_header->GetLoopInformation() == nullptr) || + !old_pre_header->GetLoopInformation()->IsBackEdge(*old_pre_header)); + UpdateLoopAndTryInformationOfNewBlock( + if_block, old_pre_header, /* replace_if_back_edge */ false); + UpdateLoopAndTryInformationOfNewBlock( + true_block, old_pre_header, /* replace_if_back_edge */ false); + UpdateLoopAndTryInformationOfNewBlock( + false_block, old_pre_header, /* replace_if_back_edge */ false); + UpdateLoopAndTryInformationOfNewBlock( + new_pre_header, old_pre_header, /* replace_if_back_edge */ false); } static void CheckAgainstUpperBound(ReferenceTypeInfo rti, ReferenceTypeInfo upper_bound_rti) diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index 9eddfc7e0e..b684cc697f 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -353,6 +353,13 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { // and removing the invoke instruction. HInstruction* InlineInto(HGraph* outer_graph, HInvoke* invoke); + // Update the loop and try membership of `block`, which was spawned from `reference`. + // In case `reference` is a back edge, `replace_if_back_edge` notifies whether `block` + // should be the new back edge. + void UpdateLoopAndTryInformationOfNewBlock(HBasicBlock* block, + HBasicBlock* reference, + bool replace_if_back_edge); + // Need to add a couple of blocks to test if the loop body is entered and // put deoptimization instructions, etc. void TransformLoopHeaderForBCE(HBasicBlock* header); @@ -380,6 +387,7 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { } void SetCurrentInstructionId(int32_t id) { + DCHECK_GE(id, current_instruction_id_); current_instruction_id_ = id; } diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 5a9f2583fd..c1b4d2403d 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -483,7 +483,11 @@ NO_INLINE // Avoid increasing caller's frame size by large stack-allocated obje static void AllocateRegisters(HGraph* graph, CodeGenerator* codegen, PassObserver* pass_observer) { - PrepareForRegisterAllocation(graph).Run(); + { + PassScope scope(PrepareForRegisterAllocation::kPrepareForRegisterAllocationPassName, + pass_observer); + PrepareForRegisterAllocation(graph).Run(); + } SsaLivenessAnalysis liveness(graph, codegen); { PassScope scope(SsaLivenessAnalysis::kLivenessPassName, pass_observer); @@ -861,7 +865,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, const uint32_t access_flags = method->GetAccessFlags(); const InvokeType invoke_type = method->GetInvokeType(); - ArenaAllocator arena(Runtime::Current()->GetArenaPool()); + ArenaAllocator arena(Runtime::Current()->GetJitArenaPool()); CodeVectorAllocator code_allocator(&arena); std::unique_ptr<CodeGenerator> codegen; { diff --git a/compiler/optimizing/prepare_for_register_allocation.h b/compiler/optimizing/prepare_for_register_allocation.h index c8b8b0dcfa..c90724c251 100644 --- a/compiler/optimizing/prepare_for_register_allocation.h +++ b/compiler/optimizing/prepare_for_register_allocation.h @@ -32,6 +32,9 @@ class PrepareForRegisterAllocation : public HGraphDelegateVisitor { void Run(); + static constexpr const char* kPrepareForRegisterAllocationPassName = + "prepare_for_register_allocation"; + private: void VisitNullCheck(HNullCheck* check) OVERRIDE; void VisitDivZeroCheck(HDivZeroCheck* check) OVERRIDE; |