diff options
Diffstat (limited to 'compiler/optimizing')
31 files changed, 1772 insertions, 536 deletions
diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index fa6a5225e7..402eeee65f 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -1402,6 +1402,14 @@ void CodeGenerator::EmitJitRoots(uint8_t* code, entry.second = index; ++index; } + for (auto& entry : jit_class_roots_) { + // Update the `roots` with the class, and replace the address temporarily + // stored to the index in the table. + uint64_t address = entry.second; + roots->Set(index, reinterpret_cast<StackReference<mirror::Class>*>(address)->AsMirrorPtr()); + entry.second = index; + ++index; + } EmitJitRootPatches(code, roots_data); } diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index 4b11e7c699..2e2c3c00af 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -34,6 +34,7 @@ #include "stack_map_stream.h" #include "string_reference.h" #include "utils/label.h" +#include "utils/type_reference.h" namespace art { @@ -343,7 +344,7 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { void BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item); size_t ComputeStackMapsSize(); size_t GetNumberOfJitRoots() const { - return jit_string_roots_.size(); + return jit_string_roots_.size() + jit_class_roots_.size(); } // Fills the `literals` array with literals collected during code generation. @@ -611,6 +612,8 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { block_order_(nullptr), jit_string_roots_(StringReferenceValueComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_roots_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), disasm_info_(nullptr), stats_(stats), graph_(graph), @@ -681,6 +684,7 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { virtual void EmitJitRootPatches(uint8_t* code ATTRIBUTE_UNUSED, const uint8_t* roots_data ATTRIBUTE_UNUSED) { DCHECK_EQ(jit_string_roots_.size(), 0u); + DCHECK_EQ(jit_class_roots_.size(), 0u); } // Frame size required for this method. @@ -711,7 +715,12 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { // Maps a StringReference (dex_file, string_index) to the index in the literal table. // Entries are intially added with a 0 index, and `EmitJitRoots` will compute all the // indices. - ArenaSafeMap<StringReference, size_t, StringReferenceValueComparator> jit_string_roots_; + ArenaSafeMap<StringReference, uint32_t, StringReferenceValueComparator> jit_string_roots_; + + // Maps a ClassReference (dex_file, type_index) to the index in the literal table. + // Entries are intially added with a pointer in the handle zone, and `EmitJitRoots` + // will compute all the indices. + ArenaSafeMap<TypeReference, uint64_t, TypeReferenceValueComparator> jit_class_roots_; DisassemblyInformation* disasm_info_; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index ed6eef1b55..8104613d3f 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -1216,7 +1216,9 @@ CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, boot_image_address_patches_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), jit_string_patches_(StringReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Always save the LR register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(LR)); } @@ -5712,8 +5714,7 @@ HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); + case HLoadClass::LoadKind::kJitTableAddress: break; case HLoadClass::LoadKind::kDexCachePcRelative: DCHECK(!Runtime::Current()->UseJitCompilation()); @@ -5814,22 +5815,12 @@ void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); - // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives - // a 128B range. To try and reduce the number of literals if we load multiple types, - // simply split the dex cache address to a 128B aligned base loaded from a literal - // and the remaining offset embedded in the load. - static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); - DCHECK_ALIGNED(cls->GetAddress(), 4u); - constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; - uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); - uint32_t offset = address & MaxInt<uint32_t>(offset_bits); - __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); - // /* GcRoot<mirror::Class> */ out = *(base_address + offset) - GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); - generate_null_check = !cls->IsInDexCache(); + case HLoadClass::LoadKind::kJitTableAddress: { + __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetAddress())); + // /* GcRoot<mirror::Class> */ out = *out + GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -7379,10 +7370,6 @@ Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); } -Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { - return DeduplicateUint32Literal(address, &uint32_literals_); -} - Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index) { jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); @@ -7391,6 +7378,15 @@ Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); } +Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), address); + return jit_class_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( const ArenaDeque<PcRelativePatchInfo>& infos, @@ -7707,18 +7703,28 @@ void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction } } +static void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + Literal* literal, + uint64_t index_in_table) { + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = literal->GetLabel()->Position(); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint8_t* data = code + literal_offset; + reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const auto& entry : jit_string_patches_) { const auto& it = jit_string_roots_.find(entry.first); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - Literal* literal = entry.second; - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = literal->GetLabel()->Position(); - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - uint8_t* data = code + literal_offset; - reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } + for (const auto& entry : jit_class_patches_) { + const auto& it = jit_class_roots_.find(entry.first); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); } } diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index 8230512825..605169deed 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -489,8 +489,10 @@ class CodeGeneratorARM : public CodeGenerator { dex::StringIndex string_index); Literal* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, dex::TypeIndex type_index); Literal* DeduplicateBootImageAddressLiteral(uint32_t address); - Literal* DeduplicateDexCacheAddressLiteral(uint32_t address); Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index); + Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + uint64_t address); void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; @@ -599,9 +601,9 @@ class CodeGeneratorARM : public CodeGenerator { using StringToLiteralMap = ArenaSafeMap<StringReference, Literal*, StringReferenceValueComparator>; - using BootTypeToLiteralMap = ArenaSafeMap<TypeReference, - Literal*, - TypeReferenceValueComparator>; + using TypeToLiteralMap = ArenaSafeMap<TypeReference, + Literal*, + TypeReferenceValueComparator>; Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); @@ -638,7 +640,7 @@ class CodeGeneratorARM : public CodeGenerator { // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; // Deduplication map for boot type literals for kBootImageLinkTimeAddress. - BootTypeToLiteralMap boot_image_type_patches_; + TypeToLiteralMap boot_image_type_patches_; // PC-relative type patch info. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // Deduplication map for patchable boot image addresses. @@ -646,6 +648,8 @@ class CodeGeneratorARM : public CodeGenerator { // Patches for string literals in JIT compiled code. StringToLiteralMap jit_string_patches_; + // Patches for class literals in JIT compiled code. + TypeToLiteralMap jit_class_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM); }; diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 6eebd69a04..5cff303e2e 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -1162,7 +1162,9 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, boot_image_address_patches_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), jit_string_patches_(StringReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save the link register (containing the return address) to mimic Quick. AddAllocatedRegister(LocationFrom(lr)); } @@ -4169,11 +4171,6 @@ vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddres return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); } -vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral( - uint64_t address) { - return DeduplicateUint64Literal(address); -} - vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLiteral( const DexFile& dex_file, dex::StringIndex string_index) { jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); @@ -4182,6 +4179,14 @@ vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLitera [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); } +vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitClassLiteral( + const DexFile& dex_file, dex::TypeIndex type_index, uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), address); + return jit_class_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); +} + void CodeGeneratorARM64::EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg) { DCHECK(reg.IsX()); @@ -4359,7 +4364,7 @@ HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); break; case HLoadClass::LoadKind::kDexCachePcRelative: @@ -4452,26 +4457,16 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) { __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(cls->GetAddress())); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads - // that gives a 16KiB range. To try and reduce the number of literals if we load - // multiple types, simply split the dex cache address to a 16KiB aligned base - // loaded from a literal and the remaining offset embedded in the load. - static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); - DCHECK_ALIGNED(cls->GetAddress(), 4u); - constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2; - uint64_t base_address = cls->GetAddress() & ~MaxInt<uint64_t>(offset_bits); - uint32_t offset = cls->GetAddress() & MaxInt<uint64_t>(offset_bits); - __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address)); - // /* GcRoot<mirror::Class> */ out = *(base_address + offset) + case HLoadClass::LoadKind::kJitTableAddress: { + __ Ldr(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetAddress())); GenerateGcRootFieldLoad(cls, out_loc, out.X(), - offset, + /* offset */ 0, /* fixup_label */ nullptr, - read_barrier_option); - generate_null_check = !cls->IsInDexCache(); + kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -5782,17 +5777,27 @@ void InstructionCodeGeneratorARM64::VisitClassTableGet(HClassTableGet* instructi } } +static void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + vixl::aarch64::Literal<uint32_t>* literal, + uint64_t index_in_table) { + uint32_t literal_offset = literal->GetOffset(); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint8_t* data = code + literal_offset; + reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorARM64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const auto& entry : jit_string_patches_) { const auto& it = jit_string_roots_.find(entry.first); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - vixl::aarch64::Literal<uint32_t>* literal = entry.second; - uint32_t literal_offset = literal->GetOffset(); - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - uint8_t* data = code + literal_offset; - reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } + for (const auto& entry : jit_class_patches_) { + const auto& it = jit_class_roots_.find(entry.first); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); } } diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 868c8b07ed..85b6f9faf5 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -566,9 +566,11 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, dex::TypeIndex type_index); vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address); - vixl::aarch64::Literal<uint64_t>* DeduplicateDexCacheAddressLiteral(uint64_t address); vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index); + vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex string_index, + uint64_t address); void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg); void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label, @@ -682,9 +684,9 @@ class CodeGeneratorARM64 : public CodeGenerator { using StringToLiteralMap = ArenaSafeMap<StringReference, vixl::aarch64::Literal<uint32_t>*, StringReferenceValueComparator>; - using BootTypeToLiteralMap = ArenaSafeMap<TypeReference, - vixl::aarch64::Literal<uint32_t>*, - TypeReferenceValueComparator>; + using TypeToLiteralMap = ArenaSafeMap<TypeReference, + vixl::aarch64::Literal<uint32_t>*, + TypeReferenceValueComparator>; vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); @@ -733,8 +735,7 @@ class CodeGeneratorARM64 : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // Deduplication map for 64-bit literals, used for non-patchable method address, method code - // or string dex cache address. + // Deduplication map for 64-bit literals, used for non-patchable method address or method code. Uint64ToLiteralMap uint64_literals_; // Method patch info, map MethodReference to a literal for method address and method code. MethodToLiteralMap method_patches_; @@ -749,7 +750,7 @@ class CodeGeneratorARM64 : public CodeGenerator { // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; // Deduplication map for boot type literals for kBootImageLinkTimeAddress. - BootTypeToLiteralMap boot_image_type_patches_; + TypeToLiteralMap boot_image_type_patches_; // PC-relative type patch info. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // Deduplication map for patchable boot image addresses. @@ -757,6 +758,8 @@ class CodeGeneratorARM64 : public CodeGenerator { // Patches for string literals in JIT compiled code. StringToLiteralMap jit_string_patches_; + // Patches for class literals in JIT compiled code. + TypeToLiteralMap jit_class_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); }; diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 4b24ac3459..2c6df38daa 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -613,6 +613,509 @@ class ArraySetSlowPathARMVIXL : public SlowPathCodeARMVIXL { DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARMVIXL); }; +// Slow path marking an object reference `ref` during a read +// barrier. The field `obj.field` in the object `obj` holding this +// reference does not get updated by this slow path after marking (see +// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). +// +// This means that after the execution of this slow path, `ref` will +// always be up-to-date, but `obj.field` may not; i.e., after the +// flip, `ref` will be a to-space reference, but `obj.field` will +// probably still be a from-space reference (unless it gets updated by +// another thread, or if another thread installed another object +// reference (different from `ref`) in `obj.field`). +class ReadBarrierMarkSlowPathARMVIXL : public SlowPathCodeARMVIXL { + public: + ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction, + Location ref, + Location entrypoint = Location::NoLocation()) + : SlowPathCodeARMVIXL(instruction), ref_(ref), entrypoint_(entrypoint) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARMVIXL"; } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + vixl32::Register ref_reg = RegisterFrom(ref_); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg; + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsArraySet() || + instruction_->IsLoadClass() || + instruction_->IsLoadString() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast() || + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || + (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier marking slow path: " + << instruction_->DebugName(); + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. + DCHECK(!(instruction_->IsArrayGet() && + instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); + + __ Bind(GetEntryLabel()); + // No need to save live registers; it's taken care of by the + // entrypoint. Also, there is no need to update the stack mask, + // as this runtime call will not trigger a garbage collection. + CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); + DCHECK(!ref_reg.Is(sp)); + DCHECK(!ref_reg.Is(lr)); + DCHECK(!ref_reg.Is(pc)); + // IP is used internally by the ReadBarrierMarkRegX entry point + // as a temporary, it cannot be the entry point's input/output. + DCHECK(!ref_reg.Is(ip)); + DCHECK(ref_reg.IsRegister()) << ref_reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in R0): + // + // R0 <- ref + // R0 <- ReadBarrierMark(R0) + // ref <- R0 + // + // we just use rX (the register containing `ref`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + if (entrypoint_.IsValid()) { + arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); + __ Blx(RegisterFrom(entrypoint_)); + } else { + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode()); + // This runtime call does not require a stack map. + arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); + } + __ B(GetExitLabel()); + } + + private: + // The location (register) of the marked object reference. + const Location ref_; + + // The location of the entrypoint if already loaded. + const Location entrypoint_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARMVIXL); +}; + +// Slow path marking an object reference `ref` during a read barrier, +// and if needed, atomically updating the field `obj.field` in the +// object `obj` holding this reference after marking (contrary to +// ReadBarrierMarkSlowPathARM above, which never tries to update +// `obj.field`). +// +// This means that after the execution of this slow path, both `ref` +// and `obj.field` will be up-to-date; i.e., after the flip, both will +// hold the same to-space reference (unless another thread installed +// another object reference (different from `ref`) in `obj.field`). +class ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL : public SlowPathCodeARMVIXL { + public: + ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction, + Location ref, + vixl32::Register obj, + Location field_offset, + vixl32::Register temp1, + vixl32::Register temp2) + : SlowPathCodeARMVIXL(instruction), + ref_(ref), + obj_(obj), + field_offset_(field_offset), + temp1_(temp1), + temp2_(temp2) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { + return "ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL"; + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + vixl32::Register ref_reg = RegisterFrom(ref_); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg; + // This slow path is only used by the UnsafeCASObject intrinsic. + DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier marking and field updating slow path: " + << instruction_->DebugName(); + DCHECK(instruction_->GetLocations()->Intrinsified()); + DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); + DCHECK(field_offset_.IsRegisterPair()) << field_offset_; + + __ Bind(GetEntryLabel()); + + // Save the old reference. + // Note that we cannot use IP to save the old reference, as IP is + // used internally by the ReadBarrierMarkRegX entry point, and we + // need the old reference after the call to that entry point. + DCHECK(!temp1_.Is(ip)); + __ Mov(temp1_, ref_reg); + + // No need to save live registers; it's taken care of by the + // entrypoint. Also, there is no need to update the stack mask, + // as this runtime call will not trigger a garbage collection. + CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); + DCHECK(!ref_reg.Is(sp)); + DCHECK(!ref_reg.Is(lr)); + DCHECK(!ref_reg.Is(pc)); + // IP is used internally by the ReadBarrierMarkRegX entry point + // as a temporary, it cannot be the entry point's input/output. + DCHECK(!ref_reg.Is(ip)); + DCHECK(ref_reg.IsRegister()) << ref_reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in R0): + // + // R0 <- ref + // R0 <- ReadBarrierMark(R0) + // ref <- R0 + // + // we just use rX (the register containing `ref`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode()); + // This runtime call does not require a stack map. + arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); + + // If the new reference is different from the old reference, + // update the field in the holder (`*(obj_ + field_offset_)`). + // + // Note that this field could also hold a different object, if + // another thread had concurrently changed it. In that case, the + // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set + // (CAS) operation below would abort the CAS, leaving the field + // as-is. + vixl32::Label done; + __ Cmp(temp1_, ref_reg); + __ B(eq, &done); + + // Update the the holder's field atomically. This may fail if + // mutator updates before us, but it's OK. This is achieved + // using a strong compare-and-set (CAS) operation with relaxed + // memory synchronization ordering, where the expected value is + // the old reference and the desired value is the new reference. + + UseScratchRegisterScope temps(arm_codegen->GetVIXLAssembler()); + // Convenience aliases. + vixl32::Register base = obj_; + // The UnsafeCASObject intrinsic uses a register pair as field + // offset ("long offset"), of which only the low part contains + // data. + vixl32::Register offset = LowRegisterFrom(field_offset_); + vixl32::Register expected = temp1_; + vixl32::Register value = ref_reg; + vixl32::Register tmp_ptr = temps.Acquire(); // Pointer to actual memory. + vixl32::Register tmp = temp2_; // Value in memory. + + __ Add(tmp_ptr, base, offset); + + if (kPoisonHeapReferences) { + arm_codegen->GetAssembler()->PoisonHeapReference(expected); + if (value.Is(expected)) { + // Do not poison `value`, as it is the same register as + // `expected`, which has just been poisoned. + } else { + arm_codegen->GetAssembler()->PoisonHeapReference(value); + } + } + + // do { + // tmp = [r_ptr] - expected; + // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); + + vixl32::Label loop_head, exit_loop; + __ Bind(&loop_head); + + __ Ldrex(tmp, MemOperand(tmp_ptr)); + + __ Subs(tmp, tmp, expected); + + { + AssemblerAccurateScope aas(arm_codegen->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + + __ it(ne); + __ clrex(ne); + } + + __ B(ne, &exit_loop); + + __ Strex(tmp, value, MemOperand(tmp_ptr)); + __ Cmp(tmp, 1); + __ B(eq, &loop_head); + + __ Bind(&exit_loop); + + if (kPoisonHeapReferences) { + arm_codegen->GetAssembler()->UnpoisonHeapReference(expected); + if (value.Is(expected)) { + // Do not unpoison `value`, as it is the same register as + // `expected`, which has just been unpoisoned. + } else { + arm_codegen->GetAssembler()->UnpoisonHeapReference(value); + } + } + + __ Bind(&done); + __ B(GetExitLabel()); + } + + private: + // The location (register) of the marked object reference. + const Location ref_; + // The register containing the object holding the marked object reference field. + const vixl32::Register obj_; + // The location of the offset of the marked reference field within `obj_`. + Location field_offset_; + + const vixl32::Register temp1_; + const vixl32::Register temp2_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL); +}; + +// Slow path generating a read barrier for a heap reference. +class ReadBarrierForHeapReferenceSlowPathARMVIXL : public SlowPathCodeARMVIXL { + public: + ReadBarrierForHeapReferenceSlowPathARMVIXL(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) + : SlowPathCodeARMVIXL(instruction), + out_(out), + ref_(ref), + obj_(obj), + offset_(offset), + index_(index) { + DCHECK(kEmitCompilerReadBarrier); + // If `obj` is equal to `out` or `ref`, it means the initial object + // has been overwritten by (or after) the heap object reference load + // to be instrumented, e.g.: + // + // __ LoadFromOffset(kLoadWord, out, out, offset); + // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); + // + // In that case, we have lost the information about the original + // object, and the emitted read barrier cannot work properly. + DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; + DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); + LocationSummary* locations = instruction_->GetLocations(); + vixl32::Register reg_out = RegisterFrom(out_); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.GetCode())); + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast() || + (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) + << "Unexpected instruction in read barrier for heap reference slow path: " + << instruction_->DebugName(); + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. + DCHECK(!(instruction_->IsArrayGet() && + instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + // We may have to change the index's value, but as `index_` is a + // constant member (like other "inputs" of this slow path), + // introduce a copy of it, `index`. + Location index = index_; + if (index_.IsValid()) { + // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. + if (instruction_->IsArrayGet()) { + // Compute the actual memory offset and store it in `index`. + vixl32::Register index_reg = RegisterFrom(index_); + DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg.GetCode())); + if (codegen->IsCoreCalleeSaveRegister(index_reg.GetCode())) { + // We are about to change the value of `index_reg` (see the + // calls to art::arm::Thumb2Assembler::Lsl and + // art::arm::Thumb2Assembler::AddConstant below), but it has + // not been saved by the previous call to + // art::SlowPathCode::SaveLiveRegisters, as it is a + // callee-save register -- + // art::SlowPathCode::SaveLiveRegisters does not consider + // callee-save registers, as it has been designed with the + // assumption that callee-save registers are supposed to be + // handled by the called function. So, as a callee-save + // register, `index_reg` _would_ eventually be saved onto + // the stack, but it would be too late: we would have + // changed its value earlier. Therefore, we manually save + // it here into another freely available register, + // `free_reg`, chosen of course among the caller-save + // registers (as a callee-save `free_reg` register would + // exhibit the same problem). + // + // Note we could have requested a temporary register from + // the register allocator instead; but we prefer not to, as + // this is a slow path, and we know we can find a + // caller-save register that is available. + vixl32::Register free_reg = FindAvailableCallerSaveRegister(codegen); + __ Mov(free_reg, index_reg); + index_reg = free_reg; + index = LocationFrom(index_reg); + } else { + // The initial register stored in `index_` has already been + // saved in the call to art::SlowPathCode::SaveLiveRegisters + // (as it is not a callee-save register), so we can freely + // use it. + } + // Shifting the index value contained in `index_reg` by the scale + // factor (2) cannot overflow in practice, as the runtime is + // unable to allocate object arrays with a size larger than + // 2^26 - 1 (that is, 2^28 - 4 bytes). + __ Lsl(index_reg, index_reg, TIMES_4); + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + __ Add(index_reg, index_reg, offset_); + } else { + // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile + // intrinsics, `index_` is not shifted by a scale factor of 2 + // (as in the case of ArrayGet), as it is actually an offset + // to an object field within an object. + DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); + DCHECK(instruction_->GetLocations()->Intrinsified()); + DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || + (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) + << instruction_->AsInvoke()->GetIntrinsic(); + DCHECK_EQ(offset_, 0U); + DCHECK(index_.IsRegisterPair()); + // UnsafeGet's offset location is a register pair, the low + // part contains the correct offset. + index = index_.ToLow(); + } + } + + // We're moving two or three locations to locations that could + // overlap, so we need a parallel move resolver. + InvokeRuntimeCallingConventionARMVIXL calling_convention; + HParallelMove parallel_move(codegen->GetGraph()->GetArena()); + parallel_move.AddMove(ref_, + LocationFrom(calling_convention.GetRegisterAt(0)), + Primitive::kPrimNot, + nullptr); + parallel_move.AddMove(obj_, + LocationFrom(calling_convention.GetRegisterAt(1)), + Primitive::kPrimNot, + nullptr); + if (index.IsValid()) { + parallel_move.AddMove(index, + LocationFrom(calling_convention.GetRegisterAt(2)), + Primitive::kPrimInt, + nullptr); + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + } else { + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + __ Mov(calling_convention.GetRegisterAt(2), offset_); + } + arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); + CheckEntrypointTypes< + kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); + arm_codegen->Move32(out_, LocationFrom(r0)); + + RestoreLiveRegisters(codegen, locations); + __ B(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { + return "ReadBarrierForHeapReferenceSlowPathARMVIXL"; + } + + private: + vixl32::Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { + uint32_t ref = RegisterFrom(ref_).GetCode(); + uint32_t obj = RegisterFrom(obj_).GetCode(); + for (uint32_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { + if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { + return vixl32::Register(i); + } + } + // We shall never fail to find a free caller-save register, as + // there are more than two core caller-save registers on ARM + // (meaning it is possible to find one which is different from + // `ref` and `obj`). + DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); + LOG(FATAL) << "Could not find a free caller-save register"; + UNREACHABLE(); + } + + const Location out_; + const Location ref_; + const Location obj_; + const uint32_t offset_; + // An additional location containing an index to an array. + // Only used for HArrayGet and the UnsafeGetObject & + // UnsafeGetObjectVolatile intrinsics. + const Location index_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARMVIXL); +}; + +// Slow path generating a read barrier for a GC root. +class ReadBarrierForRootSlowPathARMVIXL : public SlowPathCodeARMVIXL { + public: + ReadBarrierForRootSlowPathARMVIXL(HInstruction* instruction, Location out, Location root) + : SlowPathCodeARMVIXL(instruction), out_(out), root_(root) { + DCHECK(kEmitCompilerReadBarrier); + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + vixl32::Register reg_out = RegisterFrom(out_); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.GetCode())); + DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) + << "Unexpected instruction in read barrier for GC root slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + InvokeRuntimeCallingConventionARMVIXL calling_convention; + CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); + arm_codegen->Move32(LocationFrom(calling_convention.GetRegisterAt(0)), root_); + arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, + instruction_, + instruction_->GetDexPc(), + this); + CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); + arm_codegen->Move32(out_, LocationFrom(r0)); + + RestoreLiveRegisters(codegen, locations); + __ B(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARMVIXL"; } + + private: + const Location out_; + const Location root_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARMVIXL); +}; inline vixl32::Condition ARMCondition(IfCondition cond) { switch (cond) { @@ -897,10 +1400,25 @@ void CodeGeneratorARMVIXL::GenerateFrameEntry() { GetAssembler()->cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); GetAssembler()->cfi().RelOffsetForMany(DWARFReg(s0), 0, fpu_spill_mask_, kArmWordSize); } + + if (GetGraph()->HasShouldDeoptimizeFlag()) { + UseScratchRegisterScope temps(GetVIXLAssembler()); + vixl32::Register temp = temps.Acquire(); + // Initialize should_deoptimize flag to 0. + __ Mov(temp, 0); + GetAssembler()->StoreToOffset(kStoreWord, temp, sp, -kShouldDeoptimizeFlagSize); + } + int adjust = GetFrameSize() - FrameEntrySpillSize(); __ Sub(sp, sp, adjust); GetAssembler()->cfi().AdjustCFAOffset(adjust); - GetAssembler()->StoreToOffset(kStoreWord, kMethodRegister, sp, 0); + + // Save the current method if we need it. Note that we do not + // do this in HCurrentMethod, as the instruction might have been removed + // in the SSA graph. + if (RequiresCurrentMethod()) { + GetAssembler()->StoreToOffset(kStoreWord, kMethodRegister, sp, 0); + } } void CodeGeneratorARMVIXL::GenerateFrameExit() { @@ -3103,8 +3621,7 @@ void InstructionCodeGeneratorARMVIXL::HandleLongRotate(HRor* ror) { __ And(shift_right, RegisterFrom(rhs), 0x1F); __ Lsrs(shift_left, RegisterFrom(rhs), 6); - // TODO(VIXL): Check that flags are kept after "vixl32::LeaveFlags" enabled. - __ Rsb(shift_left, shift_right, Operand::From(kArmBitsPerWord)); + __ Rsb(LeaveFlags, shift_left, shift_right, Operand::From(kArmBitsPerWord)); __ B(cc, &shift_by_32_plus_shift_right); // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). @@ -4007,7 +4524,14 @@ void InstructionCodeGeneratorARMVIXL::HandleFieldGet(HInstruction* instruction, case Primitive::kPrimNot: { // /* HeapReference<Object> */ out = *(base + offset) if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - TODO_VIXL32(FATAL); + Location temp_loc = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier call. + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, base, offset, temp_loc, /* needs_null_check */ true); + if (is_volatile) { + codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } } else { GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(out), base, offset); codegen_->MaybeRecordImplicitNullCheck(instruction); @@ -4334,7 +4858,7 @@ void LocationsBuilderARMVIXL::VisitArrayGet(HArrayGet* instruction) { LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall); if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { - TODO_VIXL32(FATAL); + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); @@ -4358,7 +4882,6 @@ void LocationsBuilderARMVIXL::VisitArrayGet(HArrayGet* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { - UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); vixl32::Register obj = InputRegisterAt(instruction, 0); @@ -4370,8 +4893,6 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { instruction->IsStringCharAt(); HInstruction* array_instr = instruction->GetArray(); bool has_intermediate_address = array_instr->IsIntermediateAddress(); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); switch (type) { case Primitive::kPrimBoolean: @@ -4412,6 +4933,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { GetAssembler()->LoadFromOffset(load_type, RegisterFrom(out_loc), obj, full_offset); } } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); if (has_intermediate_address) { @@ -4440,19 +4962,27 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { } else { codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, RegisterFrom(index)); } - temps.Release(temp); } break; } case Primitive::kPrimNot: { + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. + DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); + static_assert( sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); // /* HeapReference<Object> */ out = // *(obj + data_offset + index * sizeof(HeapReference<Object>)) if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - TODO_VIXL32(FATAL); + Location temp = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier call. + codegen_->GenerateArrayLoadWithBakerReadBarrier( + instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); } else { vixl32::Register out = OutputRegister(instruction); if (index.IsConstant()) { @@ -4470,6 +5000,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { // reference, if heap poisoning is enabled). codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); if (has_intermediate_address) { @@ -4485,7 +5016,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { __ Add(temp, obj, data_offset); } codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, RegisterFrom(index)); - temps.Release(temp); + temps.Close(); // TODO(VIXL): Use a scope to ensure that we record the pc position immediately after the // load instruction. Practically, everything is fine because the helper and VIXL, at the // time of writing, do generate the store instruction last. @@ -4506,10 +5037,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; GetAssembler()->LoadFromOffset(kLoadWordPair, LowRegisterFrom(out_loc), obj, offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, obj, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); GetAssembler()->LoadFromOffset(kLoadWordPair, LowRegisterFrom(out_loc), temp, data_offset); - temps.Release(temp); } break; } @@ -4520,10 +5051,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; GetAssembler()->LoadSFromOffset(out, obj, offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, obj, Operand(RegisterFrom(index), vixl32::LSL, TIMES_4)); GetAssembler()->LoadSFromOffset(out, temp, data_offset); - temps.Release(temp); } break; } @@ -4533,10 +5064,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; GetAssembler()->LoadDFromOffset(DRegisterFrom(out_loc), obj, offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, obj, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); GetAssembler()->LoadDFromOffset(DRegisterFrom(out_loc), temp, data_offset); - temps.Release(temp); } break; } @@ -4584,7 +5115,6 @@ void LocationsBuilderARMVIXL::VisitArraySet(HArraySet* instruction) { } void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { - UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); LocationSummary* locations = instruction->GetLocations(); vixl32::Register array = InputRegisterAt(instruction, 0); Location index = locations->InAt(1); @@ -4597,8 +5127,6 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { Location value_loc = locations->InAt(2); HInstruction* array_instr = instruction->GetArray(); bool has_intermediate_address = array_instr->IsIntermediateAddress(); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); switch (value_type) { case Primitive::kPrimBoolean: @@ -4613,6 +5141,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { StoreOperandType store_type = GetStoreOperandType(value_type); GetAssembler()->StoreToOffset(store_type, RegisterFrom(value_loc), array, full_offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); if (has_intermediate_address) { @@ -4628,7 +5157,6 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { __ Add(temp, array, data_offset); } codegen_->StoreToShiftedRegOffset(value_type, value_loc, temp, RegisterFrom(index)); - temps.Release(temp); } break; } @@ -4647,10 +5175,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { GetAssembler()->StoreToOffset(kStoreWord, value, array, offset); } else { DCHECK(index.IsRegister()) << index; + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, array, data_offset); codegen_->StoreToShiftedRegOffset(value_type, value_loc, temp, RegisterFrom(index)); - temps.Release(temp); } // TODO(VIXL): Use a scope to ensure we record the pc info immediately after the preceding // store instruction. @@ -4683,10 +5211,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { GetAssembler()->StoreToOffset(kStoreWord, value, array, offset); } else { DCHECK(index.IsRegister()) << index; + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, array, data_offset); codegen_->StoreToShiftedRegOffset(value_type, value_loc, temp, RegisterFrom(index)); - temps.Release(temp); } // TODO(VIXL): Use a scope to ensure we record the pc info immediately after the preceding // store instruction. @@ -4758,13 +5286,13 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { } else { DCHECK(index.IsRegister()) << index; + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, array, data_offset); codegen_->StoreToShiftedRegOffset(value_type, LocationFrom(source), temp, RegisterFrom(index)); - temps.Release(temp); } if (!may_need_runtime_call_for_type_check) { @@ -4793,10 +5321,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; GetAssembler()->StoreToOffset(kStoreWordPair, LowRegisterFrom(value), array, offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, array, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); GetAssembler()->StoreToOffset(kStoreWordPair, LowRegisterFrom(value), temp, data_offset); - temps.Release(temp); } break; } @@ -4808,10 +5336,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; GetAssembler()->StoreSToOffset(SRegisterFrom(value), array, offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, array, Operand(RegisterFrom(index), vixl32::LSL, TIMES_4)); GetAssembler()->StoreSToOffset(SRegisterFrom(value), temp, data_offset); - temps.Release(temp); } break; } @@ -4823,10 +5351,10 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; GetAssembler()->StoreDToOffset(DRegisterFrom(value), array, offset); } else { + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); __ Add(temp, array, Operand(RegisterFrom(index), vixl32::LSL, TIMES_8)); GetAssembler()->StoreDToOffset(DRegisterFrom(value), temp, data_offset); - temps.Release(temp); } break; } @@ -4869,8 +5397,6 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayLength(HArrayLength* instruction } void LocationsBuilderARMVIXL::VisitIntermediateAddress(HIntermediateAddress* instruction) { - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); @@ -4884,9 +5410,6 @@ void InstructionCodeGeneratorARMVIXL::VisitIntermediateAddress(HIntermediateAddr vixl32::Register first = InputRegisterAt(instruction, 0); Location second = instruction->GetLocations()->InAt(1); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); - if (second.IsRegister()) { __ Add(out, first, RegisterFrom(second)); } else { @@ -4978,7 +5501,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateSuspendCheck(HSuspendCheck* instru DCHECK_EQ(slow_path->GetSuccessor(), successor); } - UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); GetAssembler()->LoadFromOffset( kLoadUnsignedHalfword, temp, tr, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); @@ -5253,7 +5776,7 @@ HLoadClass::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kBootImageAddress: // TODO(VIXL): Enable it back when literal pools are fixed in VIXL. return HLoadClass::LoadKind::kDexCacheViaMethod; - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: // TODO(VIXL): Enable it back when literal pools are fixed in VIXL. return HLoadClass::LoadKind::kDexCacheViaMethod; case HLoadClass::LoadKind::kDexCachePcRelative: @@ -5289,7 +5812,7 @@ void LocationsBuilderARMVIXL::VisitLoadClass(HLoadClass* cls) { : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { - TODO_VIXL32(FATAL); + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } HLoadClass::LoadKind load_kind = cls->GetLoadKind(); @@ -5345,7 +5868,7 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) { TODO_VIXL32(FATAL); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { + case HLoadClass::LoadKind::kJitTableAddress: { TODO_VIXL32(FATAL); break; } @@ -6249,9 +6772,9 @@ void InstructionCodeGeneratorARMVIXL::GenerateAddLongConst(Location out, return; } __ Adds(out_low, first_low, value_low); - if (GetAssembler()->ShifterOperandCanHold(ADC, value_high, kCcKeep)) { + if (GetAssembler()->ShifterOperandCanHold(ADC, value_high, kCcDontCare)) { __ Adc(out_high, first_high, value_high); - } else if (GetAssembler()->ShifterOperandCanHold(SBC, ~value_high, kCcKeep)) { + } else if (GetAssembler()->ShifterOperandCanHold(SBC, ~value_high, kCcDontCare)) { __ Sbc(out_high, first_high, ~value_high); } else { LOG(FATAL) << "Unexpected constant " << value_high; @@ -6336,14 +6859,30 @@ void InstructionCodeGeneratorARMVIXL::HandleBitwiseOperation(HBinaryOperation* i } void InstructionCodeGeneratorARMVIXL::GenerateReferenceLoadOneRegister( - HInstruction* instruction ATTRIBUTE_UNUSED, + HInstruction* instruction, Location out, uint32_t offset, - Location maybe_temp ATTRIBUTE_UNUSED, - ReadBarrierOption read_barrier_option ATTRIBUTE_UNUSED) { + Location maybe_temp, + ReadBarrierOption read_barrier_option) { vixl32::Register out_reg = RegisterFrom(out); - if (kEmitCompilerReadBarrier) { - TODO_VIXL32(FATAL); + if (read_barrier_option == kWithReadBarrier) { + CHECK(kEmitCompilerReadBarrier); + DCHECK(maybe_temp.IsRegister()) << maybe_temp; + if (kUseBakerReadBarrier) { + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // Save the value of `out` into `maybe_temp` before overwriting it + // in the following move operation, as we will need it for the + // read barrier below. + __ Mov(RegisterFrom(maybe_temp), out_reg); + // /* HeapReference<Object> */ out = *(out + offset) + GetAssembler()->LoadFromOffset(kLoadWord, out_reg, out_reg, offset); + codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); + } } else { // Plain load with no read barrier. // /* HeapReference<Object> */ out = *(out + offset) @@ -6353,16 +6892,28 @@ void InstructionCodeGeneratorARMVIXL::GenerateReferenceLoadOneRegister( } void InstructionCodeGeneratorARMVIXL::GenerateReferenceLoadTwoRegisters( - HInstruction* instruction ATTRIBUTE_UNUSED, + HInstruction* instruction, Location out, Location obj, uint32_t offset, - Location maybe_temp ATTRIBUTE_UNUSED, - ReadBarrierOption read_barrier_option ATTRIBUTE_UNUSED) { + Location maybe_temp, + ReadBarrierOption read_barrier_option) { vixl32::Register out_reg = RegisterFrom(out); vixl32::Register obj_reg = RegisterFrom(obj); - if (kEmitCompilerReadBarrier) { - TODO_VIXL32(FATAL); + if (read_barrier_option == kWithReadBarrier) { + CHECK(kEmitCompilerReadBarrier); + if (kUseBakerReadBarrier) { + DCHECK(maybe_temp.IsRegister()) << maybe_temp; + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + GetAssembler()->LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); + codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); + } } else { // Plain load with no read barrier. // /* HeapReference<Object> */ out = *(obj + offset) @@ -6372,14 +6923,61 @@ void InstructionCodeGeneratorARMVIXL::GenerateReferenceLoadTwoRegisters( } void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( - HInstruction* instruction ATTRIBUTE_UNUSED, + HInstruction* instruction, Location root, vixl32::Register obj, uint32_t offset, ReadBarrierOption read_barrier_option) { vixl32::Register root_reg = RegisterFrom(root); if (read_barrier_option == kWithReadBarrier) { - TODO_VIXL32(FATAL); + DCHECK(kEmitCompilerReadBarrier); + if (kUseBakerReadBarrier) { + // Fast path implementation of art::ReadBarrier::BarrierForRoot when + // Baker's read barrier are used: + // + // root = obj.field; + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // if (temp != null) { + // root = temp(root) + // } + + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset); + static_assert( + sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), + "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " + "have different sizes."); + static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::CompressedReference<mirror::Object> and int32_t " + "have different sizes."); + + // Slow path marking the GC root `root`. + Location temp = LocationFrom(lr); + SlowPathCodeARMVIXL* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL( + instruction, + root, + /*entrypoint*/ temp); + codegen_->AddSlowPath(slow_path); + + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + const int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); + // Loading the entrypoint does not require a load acquire since it is only changed when + // threads are suspended or running a checkpoint. + GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, entry_point_offset); + // The entrypoint is null when the GC is not marking, this prevents one load compared to + // checking GetIsGcMarking. + __ CompareAndBranchIfNonZero(RegisterFrom(temp), slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); + } else { + // GC root loaded through a slow path for read barriers other + // than Baker's. + // /* GcRoot<mirror::Object>* */ root = obj + offset + __ Add(root_reg, obj, offset); + // /* mirror::Object* */ root = root->Read() + codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); + } } else { // Plain GC root load with no read barrier. // /* GcRoot<mirror::Object> */ root = *(obj + offset) @@ -6389,53 +6987,217 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( } } -void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier( - HInstruction* instruction ATTRIBUTE_UNUSED, - Location ref ATTRIBUTE_UNUSED, - vixl::aarch32::Register obj ATTRIBUTE_UNUSED, - uint32_t offset ATTRIBUTE_UNUSED, - Location temp ATTRIBUTE_UNUSED, - bool needs_null_check ATTRIBUTE_UNUSED) { - TODO_VIXL32(FATAL); -} +void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + vixl32::Register obj, + uint32_t offset, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // /* HeapReference<Object> */ ref = *(obj + offset) + Location no_index = Location::NoLocation(); + ScaleFactor no_scale_factor = TIMES_1; + GenerateReferenceLoadWithBakerReadBarrier( + instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); +} + +void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + vixl32::Register obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + // /* HeapReference<Object> */ ref = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + ScaleFactor scale_factor = TIMES_4; + GenerateReferenceLoadWithBakerReadBarrier( + instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); +} + +void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + vixl32::Register obj, + uint32_t offset, + Location index, + ScaleFactor scale_factor, + Location temp, + bool needs_null_check, + bool always_update_field, + vixl32::Register* temp2) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // In slow path based read barriers, the read barrier call is + // inserted after the original load. However, in fast path based + // Baker's read barriers, we need to perform the load of + // mirror::Object::monitor_ *before* the original reference load. + // This load-load ordering is required by the read barrier. + // The fast path/slow path (for Baker's algorithm) should look like: + // + // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); + // lfence; // Load fence or artificial data dependency to prevent load-load reordering + // HeapReference<Object> ref = *src; // Original reference load. + // bool is_gray = (rb_state == ReadBarrier::GrayState()); + // if (is_gray) { + // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. + // } + // + // Note: the original implementation in ReadBarrier::Barrier is + // slightly more complex as it performs additional checks that we do + // not do here for performance reasons. + + vixl32::Register ref_reg = RegisterFrom(ref); + vixl32::Register temp_reg = RegisterFrom(temp); + uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); + + // /* int32_t */ monitor = obj->monitor_ + GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + // /* LockWord */ lock_word = LockWord(monitor) + static_assert(sizeof(LockWord) == sizeof(int32_t), + "art::LockWord and int32_t have different sizes."); + + // Introduce a dependency on the lock_word including the rb_state, + // which shall prevent load-load reordering without using + // a memory barrier (which would be more expensive). + // `obj` is unchanged by this operation, but its value now depends + // on `temp_reg`. + __ Add(obj, obj, Operand(temp_reg, ShiftType::LSR, 32)); + + // The actual reference load. + if (index.IsValid()) { + // Load types involving an "index": ArrayGet, + // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject + // intrinsics. + // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) + if (index.IsConstant()) { + size_t computed_offset = + (Int32ConstantFrom(index) << scale_factor) + offset; + GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); + } else { + // Handle the special case of the + // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject + // intrinsics, which use a register pair as index ("long + // offset"), of which only the low part contains data. + vixl32::Register index_reg = index.IsRegisterPair() + ? LowRegisterFrom(index) + : RegisterFrom(index); + UseScratchRegisterScope temps(GetVIXLAssembler()); + const vixl32::Register temp3 = temps.Acquire(); + __ Add(temp3, obj, Operand(index_reg, ShiftType::LSL, scale_factor)); + GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, temp3, offset); + } + } else { + // /* HeapReference<Object> */ ref = *(obj + offset) + GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, offset); + } + + // Object* ref = ref_addr->AsMirrorPtr() + GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); + + // Slow path marking the object `ref` when it is gray. + SlowPathCodeARMVIXL* slow_path; + if (always_update_field) { + DCHECK(temp2 != nullptr); + // ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL only supports address + // of the form `obj + field_offset`, where `obj` is a register and + // `field_offset` is a register pair (of which only the lower half + // is used). Thus `offset` and `scale_factor` above are expected + // to be null in this code path. + DCHECK_EQ(offset, 0u); + DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); + slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL( + instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); + } else { + slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(instruction, ref); + } + AddSlowPath(slow_path); -void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier( - HInstruction* instruction ATTRIBUTE_UNUSED, - Location ref ATTRIBUTE_UNUSED, - vixl::aarch32::Register obj ATTRIBUTE_UNUSED, - uint32_t offset ATTRIBUTE_UNUSED, - Location index ATTRIBUTE_UNUSED, - ScaleFactor scale_factor ATTRIBUTE_UNUSED, - Location temp ATTRIBUTE_UNUSED, - bool needs_null_check ATTRIBUTE_UNUSED, - bool always_update_field ATTRIBUTE_UNUSED, - vixl::aarch32::Register* temp2 ATTRIBUTE_UNUSED) { - TODO_VIXL32(FATAL); + // if (rb_state == ReadBarrier::GrayState()) + // ref = ReadBarrier::Mark(ref); + // Given the numeric representation, it's enough to check the low bit of the + // rb_state. We do that by shifting the bit out of the lock word with LSRS + // which can be a 16-bit instruction unlike the TST immediate. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); + __ B(cs, slow_path->GetEntryLabel()); // Carry flag is the last bit shifted out by LSRS. + __ Bind(slow_path->GetExitLabel()); } -void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction ATTRIBUTE_UNUSED, - Location out ATTRIBUTE_UNUSED, - Location ref ATTRIBUTE_UNUSED, - Location obj ATTRIBUTE_UNUSED, - uint32_t offset ATTRIBUTE_UNUSED, - Location index ATTRIBUTE_UNUSED) { - TODO_VIXL32(FATAL); +void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { + DCHECK(kEmitCompilerReadBarrier); + + // Insert a slow path based read barrier *after* the reference load. + // + // If heap poisoning is enabled, the unpoisoning of the loaded + // reference will be carried out by the runtime within the slow + // path. + // + // Note that `ref` currently does not get unpoisoned (when heap + // poisoning is enabled), which is alright as the `ref` argument is + // not used by the artReadBarrierSlow entry point. + // + // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. + SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) + ReadBarrierForHeapReferenceSlowPathARMVIXL(instruction, out, ref, obj, offset, index); + AddSlowPath(slow_path); + + __ B(slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); } -void CodeGeneratorARMVIXL::MaybeGenerateReadBarrierSlow(HInstruction* instruction ATTRIBUTE_UNUSED, +void CodeGeneratorARMVIXL::MaybeGenerateReadBarrierSlow(HInstruction* instruction, Location out, - Location ref ATTRIBUTE_UNUSED, - Location obj ATTRIBUTE_UNUSED, - uint32_t offset ATTRIBUTE_UNUSED, - Location index ATTRIBUTE_UNUSED) { + Location ref, + Location obj, + uint32_t offset, + Location index) { if (kEmitCompilerReadBarrier) { + // Baker's read barriers shall be handled by the fast path + // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). DCHECK(!kUseBakerReadBarrier); - TODO_VIXL32(FATAL); + // If heap poisoning is enabled, unpoisoning will be taken care of + // by the runtime within the slow path. + GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); } else if (kPoisonHeapReferences) { GetAssembler()->UnpoisonHeapReference(RegisterFrom(out)); } } +void CodeGeneratorARMVIXL::GenerateReadBarrierForRootSlow(HInstruction* instruction, + Location out, + Location root) { + DCHECK(kEmitCompilerReadBarrier); + + // Insert a slow path based read barrier *after* the GC root load. + // + // Note that GC roots are not affected by heap poisoning, so we do + // not need to do anything special for this here. + SlowPathCodeARMVIXL* slow_path = + new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARMVIXL(instruction, out, root); + AddSlowPath(slow_path); + + __ B(slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARMVIXL::GetSupportedInvokeStaticOrDirectDispatch( @@ -6805,7 +7567,7 @@ void InstructionCodeGeneratorARMVIXL::VisitPackedSwitch(HPackedSwitch* switch_in if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->GetVIXLAssembler()->IsUsingT32()) { // Create a series of compare/jumps. - UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp_reg = temps.Acquire(); // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store // the immediate, because IP is used as the destination register. For the other @@ -6853,7 +7615,7 @@ void InstructionCodeGeneratorARMVIXL::VisitPackedSwitch(HPackedSwitch* switch_in __ Cmp(key_reg, num_entries - 1); __ B(hi, codegen_->GetLabelOf(default_block)); - UseScratchRegisterScope temps(GetAssembler()->GetVIXLAssembler()); + UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register jump_offset = temps.Acquire(); // Load jump offset from the table. diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index b7ba8ddf0d..5ec3da4652 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -576,7 +576,15 @@ class CodeGeneratorARMVIXL : public CodeGenerator { uint32_t offset, Location temp, bool needs_null_check); - + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference array load when Baker's read barriers are used. + void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + vixl::aarch32::Register obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check); // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier, // GenerateArrayLoadWithBakerReadBarrier and some intrinsics. // @@ -634,6 +642,19 @@ class CodeGeneratorARMVIXL : public CodeGenerator { uint32_t offset, Location index = Location::NoLocation()); + // Generate a read barrier for a GC root within `instruction` using + // a slow path. + // + // A read barrier for an object reference GC root is implemented as + // a call to the artReadBarrierForRootSlow runtime entry point, + // which is passed the value in location `root`: + // + // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root); + // + // The `out` location contains the value returned by + // artReadBarrierForRootSlow. + void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); + void GenerateNop() OVERRIDE; void GenerateImplicitNullCheck(HNullCheck* instruction) OVERRIDE; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 61dabfabaa..cae4161daf 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -99,8 +99,9 @@ Location InvokeDexCallingConventionVisitorMIPS::GetNextLocation(Primitive::Type uint32_t gp_index = gp_index_; gp_index_ += 2; if (gp_index + 1 < calling_convention.GetNumberOfRegisters()) { - if (calling_convention.GetRegisterAt(gp_index) == A1) { - gp_index_++; // Skip A1, and use A2_A3 instead. + Register reg = calling_convention.GetRegisterAt(gp_index); + if (reg == A1 || reg == A3) { + gp_index_++; // Skip A1(A3), and use A2_A3(T0_T1) instead. gp_index++; } Register low_even = calling_convention.GetRegisterAt(gp_index); @@ -5095,9 +5096,9 @@ void LocationsBuilderMIPS::HandleInvoke(HInvoke* invoke) { void LocationsBuilderMIPS::VisitInvokeInterface(HInvokeInterface* invoke) { HandleInvoke(invoke); - // The register T0 is required to be used for the hidden argument in + // The register T7 is required to be used for the hidden argument in // art_quick_imt_conflict_trampoline, so add the hidden argument. - invoke->GetLocations()->AddTemp(Location::RegisterLocation(T0)); + invoke->GetLocations()->AddTemp(Location::RegisterLocation(T7)); } void InstructionCodeGeneratorMIPS::VisitInvokeInterface(HInvokeInterface* invoke) { @@ -5250,9 +5251,9 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - fallback_load = false; + fallback_load = true; break; case HLoadClass::LoadKind::kDexCachePcRelative: DCHECK(!Runtime::Current()->UseJitCompilation()); @@ -5613,17 +5614,8 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) { codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); - static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); - DCHECK_ALIGNED(cls->GetAddress(), 4u); - int16_t offset = Low16Bits(address); - uint32_t base_address = address - offset; // This accounts for offset sign extension. - __ Lui(out, High16Bits(base_address)); - // /* GcRoot<mirror::Class> */ out = *(base_address + offset) - GenerateGcRootFieldLoad(cls, out_loc, out, offset); - generate_null_check = !cls->IsInDexCache(); + case HLoadClass::LoadKind::kJitTableAddress: { + LOG(FATAL) << "Unimplemented"; break; } case HLoadClass::LoadKind::kDexCachePcRelative: { diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index 2273e52b06..f03f29c5d4 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -32,11 +32,11 @@ namespace mips { // InvokeDexCallingConvention registers static constexpr Register kParameterCoreRegisters[] = - { A1, A2, A3 }; + { A1, A2, A3, T0, T1 }; static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); static constexpr FRegister kParameterFpuRegisters[] = - { F12, F14 }; + { F8, F10, F12, F14, F16, F18 }; static constexpr size_t kParameterFpuRegistersLength = arraysize(kParameterFpuRegisters); @@ -48,7 +48,7 @@ static constexpr size_t kRuntimeParameterCoreRegistersLength = arraysize(kRuntimeParameterCoreRegisters); static constexpr FRegister kRuntimeParameterFpuRegisters[] = - { F12, F14}; + { F12, F14 }; static constexpr size_t kRuntimeParameterFpuRegistersLength = arraysize(kRuntimeParameterFpuRegisters); diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index d6e92ccb81..8612a67c8b 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -1013,6 +1013,7 @@ CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), constant_area_start_(-1), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), method_address_offset_(-1) { @@ -6034,7 +6035,7 @@ HLoadClass::LoadKind CodeGeneratorX86::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); break; case HLoadClass::LoadKind::kDexCacheViaMethod: @@ -6073,6 +6074,16 @@ void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) { locations->SetOut(Location::RequiresRegister()); } +Label* CodeGeneratorX86::NewJitRootClassPatch(const DexFile& dex_file, + dex::TypeIndex dex_index, + uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, dex_index), address); + // Add a patch entry and return the label. + jit_class_patches_.emplace_back(dex_file, dex_index.index_); + PatchInfo<Label>* info = &jit_class_patches_.back(); + return &info->label; +} + void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) { LocationSummary* locations = cls->GetLocations(); if (cls->NeedsAccessCheck()) { @@ -6124,16 +6135,12 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) { codegen_->RecordSimplePatch(); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); + case HLoadClass::LoadKind::kJitTableAddress: { + Address address = Address::Absolute(CodeGeneratorX86::kDummy32BitOffset); + Label* fixup_label = codegen_->NewJitRootClassPatch( + cls->GetDexFile(), cls->GetTypeIndex(), cls->GetAddress()); // /* GcRoot<mirror::Class> */ out = *address - GenerateGcRootFieldLoad(cls, - out_loc, - Address::Absolute(address), - /* fixup_label */ nullptr, - read_barrier_option); - generate_null_check = !cls->IsInDexCache(); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -7770,18 +7777,31 @@ void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type t } } +void CodeGeneratorX86::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const { + uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; + reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = + dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorX86::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const PatchInfo<Label>& info : jit_string_patches_) { - const auto& it = jit_string_roots_.find(StringReference(&info.dex_file, - dex::StringIndex(info.index))); + const auto& it = jit_string_roots_.find( + StringReference(&info.dex_file, dex::StringIndex(info.index))); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; - reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = - dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, info, it->second); + } + + for (const PatchInfo<Label>& info : jit_class_patches_) { + const auto& it = jit_class_roots_.find( + TypeReference(&info.dex_file, dex::TypeIndex(info.index))); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); } } diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 2ae3670bed..c44da97a90 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -416,12 +416,17 @@ class CodeGeneratorX86 : public CodeGenerator { Label* NewStringBssEntryPatch(HLoadString* load_string); Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index); + Label* NewJitRootClassPatch(const DexFile& dex_file, dex::TypeIndex dex_index, uint64_t address); void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const; void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; // Emit a write barrier. @@ -623,6 +628,9 @@ class CodeGeneratorX86 : public CodeGenerator { // Patches for string root accesses in JIT compiled code. ArenaDeque<PatchInfo<Label>> jit_string_patches_; + // Patches for class root accesses in JIT compiled code. + ArenaDeque<PatchInfo<Label>> jit_class_patches_; + // Offset to the start of the constant area in the assembled code. // Used for fixups to the constant area. int32_t constant_area_start_; diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 4474decf59..7dfc736d9c 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -1260,7 +1260,8 @@ CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph, string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister)); } @@ -5460,8 +5461,7 @@ HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); + case HLoadClass::LoadKind::kJitTableAddress: break; case HLoadClass::LoadKind::kDexCachePcRelative: DCHECK(!Runtime::Current()->UseJitCompilation()); @@ -5500,6 +5500,16 @@ void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) { locations->SetOut(Location::RequiresRegister()); } +Label* CodeGeneratorX86_64::NewJitRootClassPatch(const DexFile& dex_file, + dex::TypeIndex dex_index, + uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, dex_index), address); + // Add a patch entry and return the label. + jit_class_patches_.emplace_back(dex_file, dex_index.index_); + PatchInfo<Label>* info = &jit_class_patches_.back(); + return &info->label; +} + void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) { LocationSummary* locations = cls->GetLocations(); if (cls->NeedsAccessCheck()) { @@ -5543,26 +5553,13 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) { codegen_->RecordSimplePatch(); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); + case HLoadClass::LoadKind::kJitTableAddress: { + Address address = Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, + /* no_rip */ true); + Label* fixup_label = + codegen_->NewJitRootClassPatch(cls->GetDexFile(), cls->GetTypeIndex(), cls->GetAddress()); // /* GcRoot<mirror::Class> */ out = *address - if (IsUint<32>(cls->GetAddress())) { - Address address = Address::Absolute(cls->GetAddress(), /* no_rip */ true); - GenerateGcRootFieldLoad(cls, - out_loc, - address, - /* fixup_label */ nullptr, - read_barrier_option); - } else { - // TODO: Consider using opcode A1, i.e. movl eax, moff32 (with 64-bit address). - __ movq(out, Immediate(cls->GetAddress())); - GenerateGcRootFieldLoad(cls, - out_loc, - Address(out, 0), - /* fixup_label */ nullptr, - read_barrier_option); - } - generate_null_check = !cls->IsInDexCache(); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -7127,18 +7124,31 @@ void CodeGeneratorX86_64::MoveInt64ToAddress(const Address& addr_low, } } +void CodeGeneratorX86_64::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const { + uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; + reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = + dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorX86_64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const PatchInfo<Label>& info : jit_string_patches_) { - const auto& it = jit_string_roots_.find(StringReference(&info.dex_file, - dex::StringIndex(info.index))); + const auto& it = jit_string_roots_.find( + StringReference(&info.dex_file, dex::StringIndex(info.index))); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; - reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = - dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, info, it->second); + } + + for (const PatchInfo<Label>& info : jit_class_patches_) { + const auto& it = jit_class_roots_.find( + TypeReference(&info.dex_file, dex::TypeIndex(info.index))); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); } } diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 2f41f73da6..391a23b7ce 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -413,11 +413,17 @@ class CodeGeneratorX86_64 : public CodeGenerator { Label* NewStringBssEntryPatch(HLoadString* load_string); Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index); + Label* NewJitRootClassPatch(const DexFile& dex_file, dex::TypeIndex dex_index, uint64_t address); void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; const X86_64InstructionSetFeatures& GetInstructionSetFeatures() const { @@ -608,6 +614,9 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Patches for string literals in JIT compiled code. ArenaDeque<PatchInfo<Label>> jit_string_patches_; + // Patches for class literals in JIT compiled code. + ArenaDeque<PatchInfo<Label>> jit_class_patches_; + DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86_64); }; diff --git a/compiler/optimizing/emit_swap_mips_test.cc b/compiler/optimizing/emit_swap_mips_test.cc index 9dc53e6811..0d4e1c5c97 100644 --- a/compiler/optimizing/emit_swap_mips_test.cc +++ b/compiler/optimizing/emit_swap_mips_test.cc @@ -154,54 +154,54 @@ TEST_F(EmitSwapMipsTest, TwoRegisterPairs) { TEST_F(EmitSwapMipsTest, TwoFpuRegistersFloat) { moves_->AddMove( Location::FpuRegisterLocation(4), - Location::FpuRegisterLocation(6), + Location::FpuRegisterLocation(2), Primitive::kPrimFloat, nullptr); moves_->AddMove( - Location::FpuRegisterLocation(6), + Location::FpuRegisterLocation(2), Location::FpuRegisterLocation(4), Primitive::kPrimFloat, nullptr); const char* expected = - "mov.s $f8, $f6\n" - "mov.s $f6, $f4\n" - "mov.s $f4, $f8\n"; + "mov.s $f6, $f2\n" + "mov.s $f2, $f4\n" + "mov.s $f4, $f6\n"; DriverWrapper(moves_, expected, "TwoFpuRegistersFloat"); } TEST_F(EmitSwapMipsTest, TwoFpuRegistersDouble) { moves_->AddMove( Location::FpuRegisterLocation(4), - Location::FpuRegisterLocation(6), + Location::FpuRegisterLocation(2), Primitive::kPrimDouble, nullptr); moves_->AddMove( - Location::FpuRegisterLocation(6), + Location::FpuRegisterLocation(2), Location::FpuRegisterLocation(4), Primitive::kPrimDouble, nullptr); const char* expected = - "mov.d $f8, $f6\n" - "mov.d $f6, $f4\n" - "mov.d $f4, $f8\n"; + "mov.d $f6, $f2\n" + "mov.d $f2, $f4\n" + "mov.d $f4, $f6\n"; DriverWrapper(moves_, expected, "TwoFpuRegistersDouble"); } TEST_F(EmitSwapMipsTest, RegisterAndFpuRegister) { moves_->AddMove( Location::RegisterLocation(4), - Location::FpuRegisterLocation(6), + Location::FpuRegisterLocation(2), Primitive::kPrimFloat, nullptr); moves_->AddMove( - Location::FpuRegisterLocation(6), + Location::FpuRegisterLocation(2), Location::RegisterLocation(4), Primitive::kPrimFloat, nullptr); const char* expected = "or $t8, $a0, $zero\n" - "mfc1 $a0, $f6\n" - "mtc1 $t8, $f6\n"; + "mfc1 $a0, $f2\n" + "mtc1 $t8, $f2\n"; DriverWrapper(moves_, expected, "RegisterAndFpuRegister"); } @@ -327,9 +327,9 @@ TEST_F(EmitSwapMipsTest, FpuRegisterAndStackSlot) { Primitive::kPrimFloat, nullptr); const char* expected = - "mov.s $f8, $f4\n" + "mov.s $f6, $f4\n" "lwc1 $f4, 48($sp)\n" - "swc1 $f8, 48($sp)\n"; + "swc1 $f6, 48($sp)\n"; DriverWrapper(moves_, expected, "FpuRegisterAndStackSlot"); } @@ -345,9 +345,9 @@ TEST_F(EmitSwapMipsTest, FpuRegisterAndDoubleStackSlot) { Primitive::kPrimDouble, nullptr); const char* expected = - "mov.d $f8, $f4\n" + "mov.d $f6, $f4\n" "ldc1 $f4, 48($sp)\n" - "sdc1 $f8, 48($sp)\n"; + "sdc1 $f6, 48($sp)\n"; DriverWrapper(moves_, expected, "FpuRegisterAndDoubleStackSlot"); } diff --git a/compiler/optimizing/induction_var_analysis.cc b/compiler/optimizing/induction_var_analysis.cc index 15615022e3..c240c67e79 100644 --- a/compiler/optimizing/induction_var_analysis.cc +++ b/compiler/optimizing/induction_var_analysis.cc @@ -296,21 +296,22 @@ void HInductionVarAnalysis::ClassifyNonTrivial(HLoopInformation* loop) { update = SolveAddSub( loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kSub, true); } else if (instruction->IsMul()) { - update = SolveGeo( + update = SolveOp( loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kMul); } else if (instruction->IsDiv()) { - update = SolveGeo( + update = SolveOp( loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kDiv); } else if (instruction->IsRem()) { - update = SolveGeo( - loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kNop); + update = SolveOp( + loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kRem); } else if (instruction->IsShl()) { HInstruction* mulc = GetMultConstantForShift(loop, instruction); if (mulc != nullptr) { - update = SolveGeo(loop, phi, instruction, instruction->InputAt(0), mulc, kMul); + update = SolveOp(loop, phi, instruction, instruction->InputAt(0), mulc, kMul); } } else if (instruction->IsXor()) { - update = SolveXor(loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1)); + update = SolveOp( + loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kXor); } else if (instruction->IsEqual()) { update = SolveTest(loop, phi, instruction, 0); } else if (instruction->IsNotEqual()) { @@ -334,6 +335,7 @@ void HInductionVarAnalysis::ClassifyNonTrivial(HLoopInformation* loop) { FALLTHROUGH_INTENDED; case kPolynomial: case kGeometric: + case kWrapAround: // Classify first phi and then the rest of the cycle "on-demand". // Statements are scanned in order. AssignInfo(loop, phi, induction); @@ -402,14 +404,15 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferAddSub(Indu InductionOp op) { // Transfer over an addition or subtraction: any invariant, linear, polynomial, geometric, // wrap-around, or periodic can be combined with an invariant to yield a similar result. - // Even two linear inputs can be combined. Other combinations fail. + // Two linear or two polynomial inputs can be combined too. Other combinations fail. if (a != nullptr && b != nullptr) { type_ = Narrowest(type_, Narrowest(a->type, b->type)); if (a->induction_class == kInvariant && b->induction_class == kInvariant) { return CreateInvariantOp(op, a, b); - } else if (a->induction_class == kLinear && b->induction_class == kLinear) { - return CreateInduction(kLinear, - kNop, + } else if ((a->induction_class == kLinear && b->induction_class == kLinear) || + (a->induction_class == kPolynomial && b->induction_class == kPolynomial)) { + return CreateInduction(a->induction_class, + a->operation, TransferAddSub(a->op_a, b->op_a, op), TransferAddSub(a->op_b, b->op_b, op), /*fetch*/ nullptr, @@ -555,18 +558,33 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveAddSub(HLoopIn HInstruction* y, InductionOp op, bool is_first_call) { - // Solve within a cycle over an addition or subtraction: adding or subtracting an - // invariant value, seeded from phi, keeps adding to the stride of the linear induction. + // Solve within a cycle over an addition or subtraction. InductionInfo* b = LookupInfo(loop, y); - if (b != nullptr && b->induction_class == kInvariant) { - if (x == entry_phi) { - return (op == kAdd) ? b : CreateInvariantOp(kNeg, nullptr, b); - } - auto it = cycle_.find(x); - if (it != cycle_.end()) { - InductionInfo* a = it->second; - if (a->induction_class == kInvariant) { - return CreateInvariantOp(op, a, b); + if (b != nullptr) { + if (b->induction_class == kInvariant) { + // Adding or subtracting an invariant value, seeded from phi, + // keeps adding to the stride of the linear induction. + if (x == entry_phi) { + return (op == kAdd) ? b : CreateInvariantOp(kNeg, nullptr, b); + } + auto it = cycle_.find(x); + if (it != cycle_.end()) { + InductionInfo* a = it->second; + if (a->induction_class == kInvariant) { + return CreateInvariantOp(op, a, b); + } + } + } else if (op == kAdd && b->induction_class == kLinear) { + // Solve within a tight cycle that adds a term that is already classified as a linear + // induction for a polynomial induction k = k + i (represented as sum over linear terms). + if (x == entry_phi && entry_phi->InputCount() == 2 && instruction == entry_phi->InputAt(1)) { + InductionInfo* initial = LookupInfo(loop, entry_phi->InputAt(0)); + return CreateInduction(kPolynomial, + kNop, + b, + initial, + /*fetch*/ nullptr, + type_); } } } @@ -593,58 +611,63 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveAddSub(HLoopIn } } } - return nullptr; } -HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveGeo(HLoopInformation* loop, +HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveOp(HLoopInformation* loop, HInstruction* entry_phi, HInstruction* instruction, HInstruction* x, HInstruction* y, InductionOp op) { - // Solve within a tight cycle that is formed by exactly two instructions, one phi and - // one update, for a geometric induction of the form k = k * c. - if (x == entry_phi && entry_phi->InputCount() == 2 && instruction == entry_phi->InputAt(1)) { - InductionInfo* initial = LookupInfo(loop, entry_phi->InputAt(0)); - InductionInfo* b = LookupInfo(loop, y); - if (b != nullptr && b->induction_class == kInvariant && b->operation == kFetch) { - return CreateInduction(kGeometric, - op, - initial, - CreateConstant(0, type_), - b->fetch, - type_); - } - } - return nullptr; -} - -HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveXor(HLoopInformation* loop, - HInstruction* entry_phi, - HInstruction* instruction, - HInstruction* x, - HInstruction* y) { - // Solve within a tight cycle on periodic idiom of the form x = c ^ x or x = x ^ c. + // Solve within a tight cycle for a binary operation k = k op c or, for some op, k = c op k. if (entry_phi->InputCount() == 2 && instruction == entry_phi->InputAt(1)) { - InductionInfo* initial = LookupInfo(loop, entry_phi->InputAt(0)); - InductionInfo* a = LookupInfo(loop, x); - if (a != nullptr && a->induction_class == kInvariant && entry_phi == y) { - return CreateInduction(kPeriodic, - kNop, - CreateInvariantOp(kXor, a, initial), - initial, - /*fetch*/ nullptr, - type_); - } + InductionInfo* c = nullptr; InductionInfo* b = LookupInfo(loop, y); if (b != nullptr && b->induction_class == kInvariant && entry_phi == x) { - return CreateInduction(kPeriodic, - kNop, - CreateInvariantOp(kXor, initial, b), - initial, - /*fetch*/ nullptr, - type_); + c = b; + } else if (op != kDiv && op != kRem) { + InductionInfo* a = LookupInfo(loop, x); + if (a != nullptr && a->induction_class == kInvariant && entry_phi == y) { + c = a; + } + } + // Found suitable operand left or right? + if (c != nullptr) { + InductionInfo* initial = LookupInfo(loop, entry_phi->InputAt(0)); + switch (op) { + case kMul: + case kDiv: + // Restrict base of geometric induction to direct fetch. + if (c->operation == kFetch) { + return CreateInduction(kGeometric, + op, + initial, + CreateConstant(0, type_), + c->fetch, + type_); + }; + break; + case kRem: + // Idiomatic MOD wrap-around induction. + return CreateInduction(kWrapAround, + kNop, + initial, + CreateInvariantOp(kRem, initial, c), + /*fetch*/ nullptr, + type_); + case kXor: + // Idiomatic XOR periodic induction. + return CreateInduction(kPeriodic, + kNop, + CreateInvariantOp(kXor, initial, c), + initial, + /*fetch*/ nullptr, + type_); + default: + CHECK(false) << op; + break; + } } } return nullptr; @@ -659,9 +682,9 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveTest(HLoopInfo HInstruction* x = instruction->InputAt(0); HInstruction* y = instruction->InputAt(1); if (IsExact(LookupInfo(loop, x), &value) && value == opposite_value) { - return SolveXor(loop, entry_phi, instruction, graph_->GetIntConstant(1), y); + return SolveOp(loop, entry_phi, instruction, graph_->GetIntConstant(1), y, kXor); } else if (IsExact(LookupInfo(loop, y), &value) && value == opposite_value) { - return SolveXor(loop, entry_phi, instruction, x, graph_->GetIntConstant(1)); + return SolveOp(loop, entry_phi, instruction, x, graph_->GetIntConstant(1), kXor); } return nullptr; } @@ -1018,7 +1041,7 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::CreateSimplifiedInv HInstruction* HInductionVarAnalysis::GetMultConstantForShift(HLoopInformation* loop, HInstruction* instruction) { // Obtain the constant needed to treat shift as equivalent multiplication. This yields an - // existing instruction if the constants is already there. Otherwise, this has a side effect + // existing instruction if the constant is already there. Otherwise, this has a side effect // on the HIR. The restriction on the shift factor avoids generating a negative constant // (viz. 1 << 31 and 1L << 63 set the sign bit). The code assumes that generalization for // shift factors outside [0,32) and [0,64) ranges is done by earlier simplification. @@ -1102,6 +1125,7 @@ std::string HInductionVarAnalysis::InductionToString(InductionInfo* info) { case kNeg: inv += " - "; break; case kMul: inv += " * "; break; case kDiv: inv += " / "; break; + case kRem: inv += " % "; break; case kXor: inv += " ^ "; break; case kLT: inv += " < "; break; case kLE: inv += " <= "; break; @@ -1118,32 +1142,30 @@ std::string HInductionVarAnalysis::InductionToString(InductionInfo* info) { return inv; } else { if (info->induction_class == kLinear) { + DCHECK(info->operation == kNop); return "(" + InductionToString(info->op_a) + " * i + " + InductionToString(info->op_b) + "):" + Primitive::PrettyDescriptor(info->type); } else if (info->induction_class == kPolynomial) { - return "poly( sum_i(" + InductionToString(info->op_a) + ") + " + + DCHECK(info->operation == kNop); + return "poly(sum_lt(" + InductionToString(info->op_a) + ") + " + InductionToString(info->op_b) + "):" + Primitive::PrettyDescriptor(info->type); } else if (info->induction_class == kGeometric) { + DCHECK(info->operation == kMul || info->operation == kDiv); DCHECK(info->fetch != nullptr); - if (info->operation == kNop) { - return "geo(" + InductionToString(info->op_a) + " mod_i " + - FetchToString(info->fetch) + " + " + - InductionToString(info->op_b) + "):" + - Primitive::PrettyDescriptor(info->type); - } else { - return "geo(" + InductionToString(info->op_a) + " * " + - FetchToString(info->fetch) + - (info->operation == kMul ? " ^ i + " : " ^ -i + ") + - InductionToString(info->op_b) + "):" + - Primitive::PrettyDescriptor(info->type); - } + return "geo(" + InductionToString(info->op_a) + " * " + + FetchToString(info->fetch) + + (info->operation == kMul ? " ^ i + " : " ^ -i + ") + + InductionToString(info->op_b) + "):" + + Primitive::PrettyDescriptor(info->type); } else if (info->induction_class == kWrapAround) { + DCHECK(info->operation == kNop); return "wrap(" + InductionToString(info->op_a) + ", " + InductionToString(info->op_b) + "):" + Primitive::PrettyDescriptor(info->type); } else if (info->induction_class == kPeriodic) { + DCHECK(info->operation == kNop); return "periodic(" + InductionToString(info->op_a) + ", " + InductionToString(info->op_b) + "):" + Primitive::PrettyDescriptor(info->type); diff --git a/compiler/optimizing/induction_var_analysis.h b/compiler/optimizing/induction_var_analysis.h index 94afc71c3d..4720f2d61c 100644 --- a/compiler/optimizing/induction_var_analysis.h +++ b/compiler/optimizing/induction_var_analysis.h @@ -65,6 +65,7 @@ class HInductionVarAnalysis : public HOptimization { kNeg, kMul, kDiv, + kRem, kXor, kFetch, // Trip-counts. @@ -85,10 +86,10 @@ class HInductionVarAnalysis : public HOptimization { * op: a + b, a - b, -b, a * b, a / b, a % b, a ^ b, fetch * (2) linear: * nop: a * i + b - * (3) polynomial: // TODO: coming soon - * nop: sum_i(a) + b, for linear a + * (3) polynomial: + * nop: sum_lt(a) + b, for linear a * (4) geometric: - * op: a * fetch^i + b, a * fetch^-i + b, a mod_i fetch + b + * op: a * fetch^i + b, a * fetch^-i + b * (5) wrap-around * nop: a, then defined by b * (6) periodic @@ -177,17 +178,12 @@ class HInductionVarAnalysis : public HOptimization { HInstruction* y, InductionOp op, bool is_first_call); // possibly swaps x and y to try again - InductionInfo* SolveGeo(HLoopInformation* loop, - HInstruction* entry_phi, - HInstruction* instruction, - HInstruction* x, - HInstruction* y, - InductionOp op); - InductionInfo* SolveXor(HLoopInformation* loop, - HInstruction* entry_phi, - HInstruction* instruction, - HInstruction* x, - HInstruction* y); + InductionInfo* SolveOp(HLoopInformation* loop, + HInstruction* entry_phi, + HInstruction* instruction, + HInstruction* x, + HInstruction* y, + InductionOp op); InductionInfo* SolveTest(HLoopInformation* loop, HInstruction* entry_phi, HInstruction* instruction, diff --git a/compiler/optimizing/induction_var_analysis_test.cc b/compiler/optimizing/induction_var_analysis_test.cc index 2199c8e105..2d182f6483 100644 --- a/compiler/optimizing/induction_var_analysis_test.cc +++ b/compiler/optimizing/induction_var_analysis_test.cc @@ -85,6 +85,7 @@ class InductionVarAnalysisTest : public CommonCompilerTest { constant0_ = graph_->GetIntConstant(0); constant1_ = graph_->GetIntConstant(1); constant2_ = graph_->GetIntConstant(2); + constant7_ = graph_->GetIntConstant(7); constant100_ = graph_->GetIntConstant(100); float_constant0_ = graph_->GetFloatConstant(0.0f); return_->AddInstruction(new (&allocator_) HReturnVoid()); @@ -193,6 +194,7 @@ class InductionVarAnalysisTest : public CommonCompilerTest { HInstruction* constant0_; HInstruction* constant1_; HInstruction* constant2_; + HInstruction* constant7_; HInstruction* constant100_; HInstruction* float_constant0_; @@ -378,6 +380,135 @@ TEST_F(InductionVarAnalysisTest, FindTwoWayDerivedInduction) { EXPECT_TRUE(HaveSameInduction(store->InputAt(1), inc2)); } +TEST_F(InductionVarAnalysisTest, AddLinear) { + // Setup: + // for (int i = 0; i < 100; i++) { + // t1 = i + i; + // t2 = 7 + i; + // t3 = t1 + t2; + // } + BuildLoopNest(1); + + HInstruction* add1 = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, basic_[0], basic_[0]), 0); + HInstruction* add2 = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, constant7_, basic_[0]), 0); + HInstruction* add3 = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, add1, add2), 0); + PerformInductionVarAnalysis(); + + EXPECT_STREQ("((1) * i + (0)):PrimInt", GetInductionInfo(basic_[0], 0).c_str()); + EXPECT_STREQ("(((1) + (1)) * i + (0)):PrimInt", GetInductionInfo(add1, 0).c_str()); + EXPECT_STREQ("((1) * i + (7)):PrimInt", GetInductionInfo(add2, 0).c_str()); + EXPECT_STREQ("((((1) + (1)) + (1)) * i + (7)):PrimInt", GetInductionInfo(add3, 0).c_str()); +} + +TEST_F(InductionVarAnalysisTest, FindPolynomialInduction) { + // Setup: + // k = 1; + // for (int i = 0; i < 100; i++) { + // t = i * 2; + // t = 100 + t + // k = t + k; // polynomial + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(constant1_); + + HInstruction* mul = InsertInstruction( + new (&allocator_) HMul(Primitive::kPrimInt, basic_[0], constant2_), 0); + HInstruction* add = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, constant100_, mul), 0); + HInstruction* pol = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, add, k_header), 0); + k_header->AddInput(pol); + PerformInductionVarAnalysis(); + + // Note, only the phi in the cycle and the base linear induction are classified. + EXPECT_STREQ("poly(sum_lt(((2) * i + (100)):PrimInt) + (1)):PrimInt", + GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("((2) * i + (100)):PrimInt", GetInductionInfo(add, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(pol, 0).c_str()); +} + +TEST_F(InductionVarAnalysisTest, FindPolynomialInductionAndDerived) { + // Setup: + // k = 1; + // for (int i = 0; i < 100; i++) { + // t = k + 100; + // t = k - 1; + // t = - t + // t = k * 2; + // t = k << 2; + // k = k + i; // polynomial + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(constant1_); + + HInstruction* add = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, k_header, constant100_), 0); + HInstruction* sub = InsertInstruction( + new (&allocator_) HSub(Primitive::kPrimInt, k_header, constant1_), 0); + HInstruction* neg = InsertInstruction( + new (&allocator_) HNeg(Primitive::kPrimInt, sub), 0); + HInstruction* mul = InsertInstruction( + new (&allocator_) HMul(Primitive::kPrimInt, k_header, constant2_), 0); + HInstruction* shl = InsertInstruction( + new (&allocator_) HShl(Primitive::kPrimInt, k_header, constant2_), 0); + HInstruction* pol = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, k_header, basic_[0]), 0); + k_header->AddInput(pol); + PerformInductionVarAnalysis(); + + // Note, only the phi in the cycle and derived are classified. + EXPECT_STREQ("poly(sum_lt(((1) * i + (0)):PrimInt) + (1)):PrimInt", + GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("poly(sum_lt(((1) * i + (0)):PrimInt) + ((1) + (100))):PrimInt", + GetInductionInfo(add, 0).c_str()); + EXPECT_STREQ("poly(sum_lt(((1) * i + (0)):PrimInt) + ((1) - (1))):PrimInt", + GetInductionInfo(sub, 0).c_str()); + EXPECT_STREQ("poly(sum_lt((( - (1)) * i + (0)):PrimInt) + ((1) - (1))):PrimInt", + GetInductionInfo(neg, 0).c_str()); + EXPECT_STREQ("poly(sum_lt(((2) * i + (0)):PrimInt) + (2)):PrimInt", + GetInductionInfo(mul, 0).c_str()); + EXPECT_STREQ("poly(sum_lt(((4) * i + (0)):PrimInt) + (4)):PrimInt", + GetInductionInfo(shl, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(pol, 0).c_str()); +} + +TEST_F(InductionVarAnalysisTest, AddPolynomial) { + // Setup: + // k = 7; + // for (int i = 0; i < 100; i++) { + // t = k + k; + // t = t + k; + // k = k + i + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(constant7_); + + HInstruction* add1 = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, k_header, k_header), 0); + HInstruction* add2 = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, add1, k_header), 0); + HInstruction* add3 = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, k_header, basic_[0]), 0); + k_header->AddInput(add3); + PerformInductionVarAnalysis(); + + // Note, only the phi in the cycle and added-derived are classified. + EXPECT_STREQ("poly(sum_lt(((1) * i + (0)):PrimInt) + (7)):PrimInt", + GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("poly(sum_lt((((1) + (1)) * i + (0)):PrimInt) + ((7) + (7))):PrimInt", + GetInductionInfo(add1, 0).c_str()); + EXPECT_STREQ( + "poly(sum_lt(((((1) + (1)) + (1)) * i + (0)):PrimInt) + (((7) + (7)) + (7))):PrimInt", + GetInductionInfo(add2, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(add3, 0).c_str()); +} + TEST_F(InductionVarAnalysisTest, FindGeometricMulInduction) { // Setup: // k = 1; @@ -481,20 +612,20 @@ TEST_F(InductionVarAnalysisTest, FindGeometricDivInductionAndDerived) { EXPECT_STREQ("", GetInductionInfo(div, 0).c_str()); } -TEST_F(InductionVarAnalysisTest, FindGeometricRemInductionAndDerived) { +TEST_F(InductionVarAnalysisTest, FindRemWrapAroundInductionAndDerived) { // Setup: - // k = 1; + // k = 100; // for (int i = 0; i < 100; i++) { // t = k + 100; // t = k - 1; // t = -t // t = k * 2; // t = k << 2; - // k = k % 100; // geometric (% 100) + // k = k % 7; // } BuildLoopNest(1); HPhi* k_header = InsertLoopPhi(0, 0); - k_header->AddInput(constant1_); + k_header->AddInput(constant100_); HInstruction* add = InsertInstruction( new (&allocator_) HAdd(Primitive::kPrimInt, k_header, constant100_), 0); @@ -507,17 +638,17 @@ TEST_F(InductionVarAnalysisTest, FindGeometricRemInductionAndDerived) { HInstruction* shl = InsertInstruction( new (&allocator_) HShl(Primitive::kPrimInt, k_header, constant2_), 0); HInstruction* rem = InsertInstruction( - new (&allocator_) HRem(Primitive::kPrimInt, k_header, constant100_, kNoDexPc), 0); + new (&allocator_) HRem(Primitive::kPrimInt, k_header, constant7_, kNoDexPc), 0); k_header->AddInput(rem); PerformInductionVarAnalysis(); - // Note, only the phi in the cycle and direct additive derived are classified. - EXPECT_STREQ("geo((1) mod_i 100 + (0)):PrimInt", GetInductionInfo(k_header, 0).c_str()); - EXPECT_STREQ("geo((1) mod_i 100 + (100)):PrimInt", GetInductionInfo(add, 0).c_str()); - EXPECT_STREQ("geo((1) mod_i 100 + ((0) - (1))):PrimInt", GetInductionInfo(sub, 0).c_str()); - EXPECT_STREQ("", GetInductionInfo(neg, 0).c_str()); - EXPECT_STREQ("", GetInductionInfo(mul, 0).c_str()); - EXPECT_STREQ("", GetInductionInfo(shl, 0).c_str()); + // Note, only the phi in the cycle and derived are classified. + EXPECT_STREQ("wrap((100), ((100) % (7))):PrimInt", GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("wrap(((100) + (100)), (((100) % (7)) + (100))):PrimInt", GetInductionInfo(add, 0).c_str()); + EXPECT_STREQ("wrap(((100) - (1)), (((100) % (7)) - (1))):PrimInt", GetInductionInfo(sub, 0).c_str()); + EXPECT_STREQ("wrap(( - ((100) - (1))), ( - (((100) % (7)) - (1)))):PrimInt", GetInductionInfo(neg, 0).c_str()); + EXPECT_STREQ("wrap(((100) * (2)), (((100) % (7)) * (2))):PrimInt", GetInductionInfo(mul, 0).c_str()); + EXPECT_STREQ("wrap(((100) * (4)), (((100) % (7)) * (4))):PrimInt", GetInductionInfo(shl, 0).c_str()); EXPECT_STREQ("", GetInductionInfo(rem, 0).c_str()); } diff --git a/compiler/optimizing/induction_var_range.cc b/compiler/optimizing/induction_var_range.cc index 75619a3c01..e665551012 100644 --- a/compiler/optimizing/induction_var_range.cc +++ b/compiler/optimizing/induction_var_range.cc @@ -460,6 +460,8 @@ bool InductionVarRange::NeedsTripCount(HInductionVarAnalysis::InductionInfo* inf if (info != nullptr) { if (info->induction_class == HInductionVarAnalysis::kLinear) { return IsConstant(info->op_a, kExact, stride_value); + } else if (info->induction_class == HInductionVarAnalysis::kPolynomial) { + return NeedsTripCount(info->op_a, stride_value); } else if (info->induction_class == HInductionVarAnalysis::kWrapAround) { return NeedsTripCount(info->op_b, stride_value); } @@ -492,7 +494,7 @@ InductionVarRange::Value InductionVarRange::GetLinear(HInductionVarAnalysis::Ind bool in_body, bool is_min) const { DCHECK(info != nullptr); - DCHECK(info->induction_class == HInductionVarAnalysis::kLinear); + DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kLinear); // Detect common situation where an offset inside the trip-count cancels out during range // analysis (finding max a * (TC - 1) + OFFSET for a == 1 and TC = UPPER - OFFSET or finding // min a * (TC - 1) + OFFSET for a == -1 and TC = OFFSET - UPPER) to avoid losing information @@ -539,25 +541,49 @@ InductionVarRange::Value InductionVarRange::GetLinear(HInductionVarAnalysis::Ind GetVal(info->op_b, trip, in_body, is_min)); } +InductionVarRange::Value InductionVarRange::GetPolynomial(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const { + DCHECK(info != nullptr); + DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kPolynomial); + int64_t a = 0; + int64_t b = 0; + if (IsConstant(info->op_a->op_a, kExact, &a) && CanLongValueFitIntoInt(a) && a >= 0 && + IsConstant(info->op_a->op_b, kExact, &b) && CanLongValueFitIntoInt(b) && b >= 0) { + // Evaluate bounds on sum_i=0^m-1(a * i + b) + c with a,b >= 0 for known + // maximum index value m as a * (m * (m-1)) / 2 + b * m + c. + Value c = GetVal(info->op_b, trip, in_body, is_min); + if (is_min) { + return c; + } else { + Value m = GetVal(trip, trip, in_body, is_min); + Value t = DivValue(MulValue(m, SubValue(m, Value(1))), Value(2)); + Value x = MulValue(Value(a), t); + Value y = MulValue(Value(b), m); + return AddValue(AddValue(x, y), c); + } + } + return Value(); +} + InductionVarRange::Value InductionVarRange::GetGeometric(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* trip, bool in_body, bool is_min) const { DCHECK(info != nullptr); - DCHECK(info->induction_class == HInductionVarAnalysis::kGeometric); + DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kGeometric); int64_t a = 0; int64_t f = 0; if (IsConstant(info->op_a, kExact, &a) && CanLongValueFitIntoInt(a) && - IsIntAndGet(info->fetch, &f) && - CanLongValueFitIntoInt(f) && f >= 1) { - // Conservative bounds on a * f^-i + b with f >= 1 can be computed without trip count. - // Same for mod. All other forms would require a much more elaborate evaluation. + IsIntAndGet(info->fetch, &f) && f >= 1) { + // Conservative bounds on a * f^-i + b with f >= 1 can be computed without + // trip count. Other forms would require a much more elaborate evaluation. const bool is_min_a = a >= 0 ? is_min : !is_min; if (info->operation == HInductionVarAnalysis::kDiv) { - return AddValue(Value(is_min_a ? 0 : a), GetVal(info->op_b, trip, in_body, is_min)); - } else if (info->operation == HInductionVarAnalysis::kNop) { - return AddValue(Value(is_min_a ? (a % f) : a), GetVal(info->op_b, trip, in_body, is_min)); + Value b = GetVal(info->op_b, trip, in_body, is_min); + return is_min_a ? b : AddValue(Value(a), b); } } return Value(); @@ -572,7 +598,7 @@ InductionVarRange::Value InductionVarRange::GetFetch(HInstruction* instruction, if (chase_hint_ == nullptr && in_body && trip != nullptr && instruction == trip->op_a->fetch) { if (is_min) { return Value(1); - } else if (!IsUnsafeTripCount(trip)) { + } else if (!instruction->IsConstant() && !IsUnsafeTripCount(trip)) { return Value(std::numeric_limits<int32_t>::max()); } } @@ -650,6 +676,8 @@ InductionVarRange::Value InductionVarRange::GetVal(HInductionVarAnalysis::Induct return GetMul(info->op_a, info->op_b, trip, in_body, is_min); case HInductionVarAnalysis::kDiv: return GetDiv(info->op_a, info->op_b, trip, in_body, is_min); + case HInductionVarAnalysis::kRem: + return GetRem(info->op_a, info->op_b); case HInductionVarAnalysis::kXor: return GetXor(info->op_a, info->op_b); case HInductionVarAnalysis::kFetch: @@ -675,7 +703,7 @@ InductionVarRange::Value InductionVarRange::GetVal(HInductionVarAnalysis::Induct case HInductionVarAnalysis::kLinear: return CorrectForType(GetLinear(info, trip, in_body, is_min), info->type); case HInductionVarAnalysis::kPolynomial: - break; + return GetPolynomial(info, trip, in_body, is_min); case HInductionVarAnalysis::kGeometric: return GetGeometric(info, trip, in_body, is_min); case HInductionVarAnalysis::kWrapAround: @@ -757,6 +785,21 @@ InductionVarRange::Value InductionVarRange::GetDiv(HInductionVarAnalysis::Induct return Value(); } +InductionVarRange::Value InductionVarRange::GetRem( + HInductionVarAnalysis::InductionInfo* info1, + HInductionVarAnalysis::InductionInfo* info2) const { + int64_t v1 = 0; + int64_t v2 = 0; + // Only accept exact values. + if (IsConstant(info1, kExact, &v1) && IsConstant(info2, kExact, &v2) && v2 != 0) { + int64_t value = v1 % v2; + if (CanLongValueFitIntoInt(value)) { + return Value(static_cast<int32_t>(value)); + } + } + return Value(); +} + InductionVarRange::Value InductionVarRange::GetXor( HInductionVarAnalysis::InductionInfo* info1, HInductionVarAnalysis::InductionInfo* info2) const { @@ -898,8 +941,12 @@ bool InductionVarRange::GenerateRangeOrLastValue(HInstruction* context, upper = nullptr; } break; + case HInductionVarAnalysis::kPolynomial: + return GenerateLastValuePolynomial(info, trip, graph, block, lower); case HInductionVarAnalysis::kGeometric: return GenerateLastValueGeometric(info, trip, graph, block, lower); + case HInductionVarAnalysis::kWrapAround: + return GenerateLastValueWrapAround(info, trip, graph, block, lower); case HInductionVarAnalysis::kPeriodic: return GenerateLastValuePeriodic(info, trip, graph, block, lower, needs_taken_test); default: @@ -925,13 +972,43 @@ bool InductionVarRange::GenerateRangeOrLastValue(HInstruction* context, GenerateCode(info, trip, graph, block, upper, in_body, /* is_min */ false); } +bool InductionVarRange::GenerateLastValuePolynomial(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + HGraph* graph, + HBasicBlock* block, + /*out*/HInstruction** result) const { + DCHECK(info != nullptr); + DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kPolynomial); + // Detect known coefficients and trip count (always taken). + int64_t a = 0; + int64_t b = 0; + int64_t m = 0; + if (IsConstant(info->op_a->op_a, kExact, &a) && a >= 0 && + IsConstant(info->op_a->op_b, kExact, &b) && b >= 0 && + IsConstant(trip->op_a, kExact, &m) && m >= 1) { + // Evaluate bounds on sum_i=0^m-1(a * i + b) + c with a,b >= 0 for known + // maximum index value m as a * (m * (m-1)) / 2 + b * m + c. + // TODO: generalize + HInstruction* c_instr = nullptr; + if (GenerateCode(info->op_b, nullptr, graph, block, graph ? &c_instr : nullptr, false, false)) { + if (graph != nullptr) { + int64_t sum = a * ((m * (m - 1)) / 2) + b * m; + *result = Insert(block, new (graph->GetArena()) HAdd(info->type, + graph->GetIntConstant(sum), c_instr)); + } + return true; + } + } + return false; +} + bool InductionVarRange::GenerateLastValueGeometric(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* trip, HGraph* graph, HBasicBlock* block, /*out*/HInstruction** result) const { DCHECK(info != nullptr); - DCHECK(info->induction_class == HInductionVarAnalysis::kGeometric); + DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kGeometric); // Detect known base and trip count (always taken). int64_t f = 0; int64_t t = 0; @@ -940,15 +1017,6 @@ bool InductionVarRange::GenerateLastValueGeometric(HInductionVarAnalysis::Induct HInstruction* opb = nullptr; if (GenerateCode(info->op_a, nullptr, graph, block, &opa, false, false) && GenerateCode(info->op_b, nullptr, graph, block, &opb, false, false)) { - // Generate a % f + b. - if (info->operation == HInductionVarAnalysis::kNop) { - if (graph != nullptr) { - HInstruction* rem = - Insert(block, new (graph->GetArena()) HRem(info->type, opa, info->fetch, kNoDexPc)); - *result = Insert(block, new (graph->GetArena()) HAdd(info->type, rem, opb)); - } - return true; - } // Compute f ^ t. int64_t fpowt = IntPow(f, t); if (graph != nullptr) { @@ -980,6 +1048,28 @@ bool InductionVarRange::GenerateLastValueGeometric(HInductionVarAnalysis::Induct return false; } +bool InductionVarRange::GenerateLastValueWrapAround(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + HGraph* graph, + HBasicBlock* block, + /*out*/HInstruction** result) const { + DCHECK(info != nullptr); + DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kWrapAround); + // Count depth. + int32_t depth = 0; + for (; info->induction_class == HInductionVarAnalysis::kWrapAround; + info = info->op_b, ++depth) {} + // Handle wrap(x, wrap(.., y)) if trip count reaches an invariant at end. + // TODO: generalize + int64_t t = 0; + if (info->induction_class == HInductionVarAnalysis::kInvariant && + IsConstant(trip->op_a, kExact, &t) && t >= depth && + GenerateCode(info, nullptr, graph, block, result, false, false)) { + return true; + } + return false; +} + bool InductionVarRange::GenerateLastValuePeriodic(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* trip, HGraph* graph, @@ -987,17 +1077,18 @@ bool InductionVarRange::GenerateLastValuePeriodic(HInductionVarAnalysis::Inducti /*out*/HInstruction** result, /*out*/bool* needs_taken_test) const { DCHECK(info != nullptr); - DCHECK(info->induction_class == HInductionVarAnalysis::kPeriodic); + DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kPeriodic); // Count period. int32_t period = 1; for (HInductionVarAnalysis::InductionInfo* p = info; p->induction_class == HInductionVarAnalysis::kPeriodic; p = p->op_b, ++period) {} // Handle periodic(x, y) case for restricted types. + // TODO: generalize if (period != 2 || trip->op_a->type != Primitive::kPrimInt || (info->type != Primitive::kPrimInt && info->type != Primitive::kPrimBoolean)) { - return false; // TODO: easy to generalize + return false; } HInstruction* x_instr = nullptr; HInstruction* y_instr = nullptr; @@ -1058,6 +1149,7 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, // invariants, some effort is made to keep this parameter consistent). switch (info->operation) { case HInductionVarAnalysis::kAdd: + case HInductionVarAnalysis::kRem: // no proper is_min for second arg case HInductionVarAnalysis::kXor: // no proper is_min for second arg case HInductionVarAnalysis::kLT: case HInductionVarAnalysis::kLE: @@ -1070,6 +1162,8 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, switch (info->operation) { case HInductionVarAnalysis::kAdd: operation = new (graph->GetArena()) HAdd(type, opa, opb); break; + case HInductionVarAnalysis::kRem: + operation = new (graph->GetArena()) HRem(type, opa, opb, kNoDexPc); break; case HInductionVarAnalysis::kXor: operation = new (graph->GetArena()) HXor(type, opa, opb); break; case HInductionVarAnalysis::kLT: diff --git a/compiler/optimizing/induction_var_range.h b/compiler/optimizing/induction_var_range.h index f7360e83db..ba14847d82 100644 --- a/compiler/optimizing/induction_var_range.h +++ b/compiler/optimizing/induction_var_range.h @@ -190,6 +190,10 @@ class InductionVarRange { HInductionVarAnalysis::InductionInfo* trip, bool in_body, bool is_min) const; + Value GetPolynomial(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const; Value GetGeometric(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* trip, bool in_body, @@ -212,6 +216,8 @@ class InductionVarRange { HInductionVarAnalysis::InductionInfo* trip, bool in_body, bool is_min) const; + Value GetRem(HInductionVarAnalysis::InductionInfo* info1, + HInductionVarAnalysis::InductionInfo* info2) const; Value GetXor(HInductionVarAnalysis::InductionInfo* info1, HInductionVarAnalysis::InductionInfo* info2) const; @@ -249,12 +255,24 @@ class InductionVarRange { /*out*/ bool* needs_finite_test, /*out*/ bool* needs_taken_test) const; + bool GenerateLastValuePolynomial(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + HGraph* graph, + HBasicBlock* block, + /*out*/HInstruction** result) const; + bool GenerateLastValueGeometric(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* trip, HGraph* graph, HBasicBlock* block, /*out*/HInstruction** result) const; + bool GenerateLastValueWrapAround(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + HGraph* graph, + HBasicBlock* block, + /*out*/HInstruction** result) const; + bool GenerateLastValuePeriodic(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* trip, HGraph* graph, diff --git a/compiler/optimizing/induction_var_range_test.cc b/compiler/optimizing/induction_var_range_test.cc index 510841eb68..aa3e1aab4f 100644 --- a/compiler/optimizing/induction_var_range_test.cc +++ b/compiler/optimizing/induction_var_range_test.cc @@ -135,6 +135,8 @@ class InductionVarRangeTest : public CommonCompilerTest { case 'n': op = HInductionVarAnalysis::kNeg; break; case '*': op = HInductionVarAnalysis::kMul; break; case '/': op = HInductionVarAnalysis::kDiv; break; + case '%': op = HInductionVarAnalysis::kRem; break; + case '^': op = HInductionVarAnalysis::kXor; break; case '<': op = HInductionVarAnalysis::kLT; break; default: op = HInductionVarAnalysis::kNop; break; } @@ -178,12 +180,21 @@ class InductionVarRangeTest : public CommonCompilerTest { Primitive::kPrimInt); } + /** Constructs a polynomial sum(a * i + b) + c induction. */ + HInductionVarAnalysis::InductionInfo* CreatePolynomial(int32_t a, int32_t b, int32_t c) { + return iva_->CreateInduction(HInductionVarAnalysis::kPolynomial, + HInductionVarAnalysis::kNop, + CreateLinear(a, b), + CreateConst(c), + nullptr, + Primitive::kPrimInt); + } + /** Constructs a geometric a * f^i + b induction. */ HInductionVarAnalysis::InductionInfo* CreateGeometric(int32_t a, int32_t b, int32_t f, char op) { return iva_->CreateInduction(HInductionVarAnalysis::kGeometric, - op == '*' ? HInductionVarAnalysis::kMul : - op == '/' ? HInductionVarAnalysis::kDiv : - HInductionVarAnalysis::kNop, + op == '*' ? HInductionVarAnalysis::kMul + : HInductionVarAnalysis::kDiv, CreateConst(a), CreateConst(b), graph_->GetIntConstant(f), @@ -200,7 +211,7 @@ class InductionVarRangeTest : public CommonCompilerTest { Primitive::kPrimInt); } - /** Constructs a wrap-around induction consisting of a constant, followed info */ + /** Constructs a wrap-around induction consisting of a constant, followed by info. */ HInductionVarAnalysis::InductionInfo* CreateWrapAround( int32_t initial, HInductionVarAnalysis::InductionInfo* info) { @@ -256,6 +267,16 @@ class InductionVarRangeTest : public CommonCompilerTest { return range_.GetDiv(info1, info2, nullptr, /* in_body */ true, is_min); } + Value GetRem(HInductionVarAnalysis::InductionInfo* info1, + HInductionVarAnalysis::InductionInfo* info2) { + return range_.GetRem(info1, info2); + } + + Value GetXor(HInductionVarAnalysis::InductionInfo* info1, + HInductionVarAnalysis::InductionInfo* info2) { + return range_.GetXor(info1, info2); + } + bool IsExact(HInductionVarAnalysis::InductionInfo* info, int64_t* value) { return range_.IsConstant(info, InductionVarRange::kExact, value); } @@ -438,6 +459,27 @@ TEST_F(InductionVarRangeTest, GetMinMaxWrapAround) { ExpectEqual(Value(20), GetMax(CreateWrapAround(20, -1, 10), nullptr)); } +TEST_F(InductionVarRangeTest, GetMinMaxPolynomial) { + ExpectEqual(Value(7), GetMin(CreatePolynomial(3, 5, 7), nullptr)); + ExpectEqual(Value(), GetMax(CreatePolynomial(3, 5, 7), nullptr)); + ExpectEqual(Value(7), GetMin(CreatePolynomial(3, 5, 7), CreateTripCount(5, true, true))); + ExpectEqual(Value(45), GetMax(CreatePolynomial(3, 5, 7), CreateTripCount(5, true, true))); + ExpectEqual(Value(7), GetMin(CreatePolynomial(3, 5, 7), CreateTripCount(10, true, true))); + ExpectEqual(Value(160), GetMax(CreatePolynomial(3, 5, 7), CreateTripCount(10, true, true))); + ExpectEqual(Value(-7), GetMin(CreatePolynomial(11, 13, -7), + CreateTripCount(5, true, true))); + ExpectEqual(Value(111), GetMax(CreatePolynomial(11, 13, -7), + CreateTripCount(5, true, true))); + ExpectEqual(Value(-7), GetMin(CreatePolynomial(11, 13, -7), + CreateTripCount(10, true, true))); + ExpectEqual(Value(506), GetMax(CreatePolynomial(11, 13, -7), + CreateTripCount(10, true, true))); + ExpectEqual(Value(), GetMin(CreatePolynomial(-3, 5, 7), CreateTripCount(10, true, true))); + ExpectEqual(Value(), GetMax(CreatePolynomial(-3, 5, 7), CreateTripCount(10, true, true))); + ExpectEqual(Value(), GetMin(CreatePolynomial(3, -5, 7), CreateTripCount(10, true, true))); + ExpectEqual(Value(), GetMax(CreatePolynomial(3, -5, 7), CreateTripCount(10, true, true))); +} + TEST_F(InductionVarRangeTest, GetMinMaxGeometricMul) { ExpectEqual(Value(), GetMin(CreateGeometric(1, 1, 1, '*'), nullptr)); ExpectEqual(Value(), GetMax(CreateGeometric(1, 1, 1, '*'), nullptr)); @@ -454,17 +496,6 @@ TEST_F(InductionVarRangeTest, GetMinMaxGeometricDiv) { ExpectEqual(Value(-5), GetMax(CreateGeometric(-11, -5, 3, '/'), nullptr)); } -TEST_F(InductionVarRangeTest, GetMinMaxGeometricRem) { - ExpectEqual(Value(7), GetMin(CreateGeometric(11, 5, 3, '%'), nullptr)); - ExpectEqual(Value(16), GetMax(CreateGeometric(11, 5, 3, '%'), nullptr)); - ExpectEqual(Value(-3), GetMin(CreateGeometric(11, -5, 3, '%'), nullptr)); - ExpectEqual(Value(6), GetMax(CreateGeometric(11, -5, 3, '%'), nullptr)); - ExpectEqual(Value(-6), GetMin(CreateGeometric(-11, 5, 3, '%'), nullptr)); - ExpectEqual(Value(3), GetMax(CreateGeometric(-11, 5, 3, '%'), nullptr)); - ExpectEqual(Value(-16), GetMin(CreateGeometric(-11, -5, 3, '%'), nullptr)); - ExpectEqual(Value(-7), GetMax(CreateGeometric(-11, -5, 3, '%'), nullptr)); -} - TEST_F(InductionVarRangeTest, GetMinMaxPeriodic) { ExpectEqual(Value(-2), GetMin(CreateRange(-2, 99), nullptr)); ExpectEqual(Value(99), GetMax(CreateRange(-2, 99), nullptr)); @@ -530,6 +561,46 @@ TEST_F(InductionVarRangeTest, GetDivMax) { ExpectEqual(Value(), GetDiv(CreateRange(-1, 1), CreateRange(-1, 1), false)); } +TEST_F(InductionVarRangeTest, GetMinMaxRem) { + ExpectEqual(Value(), GetMin(CreateInvariant('%', CreateConst(2), CreateRange(10, 20)), nullptr)); + ExpectEqual(Value(), GetMax(CreateInvariant('%', CreateConst(2), CreateRange(10, 20)), nullptr)); + ExpectEqual(Value(), GetMin(CreateInvariant('%', CreateRange(10, 20), CreateConst(2)), nullptr)); + ExpectEqual(Value(), GetMax(CreateInvariant('%', CreateRange(10, 20), CreateConst(2)), nullptr)); + ExpectEqual(Value(2), GetMin(CreateInvariant('%', CreateConst(2), CreateConst(5)), nullptr)); + ExpectEqual(Value(2), GetMax(CreateInvariant('%', CreateConst(2), CreateConst(5)), nullptr)); + ExpectEqual(Value(1), GetMin(CreateInvariant('%', CreateConst(11), CreateConst(5)), nullptr)); + ExpectEqual(Value(1), GetMax(CreateInvariant('%', CreateConst(11), CreateConst(5)), nullptr)); +} + +TEST_F(InductionVarRangeTest, GetRem) { + ExpectEqual(Value(0), GetRem(CreateConst(1), CreateConst(1))); + ExpectEqual(Value(2), GetRem(CreateConst(2), CreateConst(5))); + ExpectEqual(Value(1), GetRem(CreateConst(11), CreateConst(5))); + ExpectEqual(Value(-2), GetRem(CreateConst(-2), CreateConst(5))); + ExpectEqual(Value(-1), GetRem(CreateConst(-11), CreateConst(5))); + ExpectEqual(Value(2), GetRem(CreateConst(2), CreateConst(-5))); + ExpectEqual(Value(1), GetRem(CreateConst(11), CreateConst(-5))); + ExpectEqual(Value(-2), GetRem(CreateConst(-2), CreateConst(-5))); + ExpectEqual(Value(-1), GetRem(CreateConst(-11), CreateConst(-5))); + ExpectEqual(Value(), GetRem(CreateConst(1), CreateConst(0))); +} + +TEST_F(InductionVarRangeTest, GetMinMaxXor) { + ExpectEqual(Value(), GetMin(CreateInvariant('^', CreateConst(2), CreateRange(10, 20)), nullptr)); + ExpectEqual(Value(), GetMax(CreateInvariant('^', CreateConst(2), CreateRange(10, 20)), nullptr)); + ExpectEqual(Value(), GetMin(CreateInvariant('^', CreateRange(10, 20), CreateConst(2)), nullptr)); + ExpectEqual(Value(), GetMax(CreateInvariant('^', CreateRange(10, 20), CreateConst(2)), nullptr)); + ExpectEqual(Value(3), GetMin(CreateInvariant('^', CreateConst(1), CreateConst(2)), nullptr)); + ExpectEqual(Value(3), GetMax(CreateInvariant('^', CreateConst(1), CreateConst(2)), nullptr)); +} + +TEST_F(InductionVarRangeTest, GetXor) { + ExpectEqual(Value(0), GetXor(CreateConst(1), CreateConst(1))); + ExpectEqual(Value(3), GetXor(CreateConst(1), CreateConst(2))); + ExpectEqual(Value(-2), GetXor(CreateConst(1), CreateConst(-1))); + ExpectEqual(Value(0), GetXor(CreateConst(-1), CreateConst(-1))); +} + TEST_F(InductionVarRangeTest, AddValue) { ExpectEqual(Value(110), AddValue(Value(10), Value(100))); ExpectEqual(Value(-5), AddValue(Value(x_, 1, -4), Value(x_, -1, -1))); diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 8d93867230..fe4662abb1 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -1444,7 +1444,7 @@ size_t HInliner::RunOptimizations(HGraph* callee_graph, // optimization that could lead to a HDeoptimize. The following optimizations do not. HDeadCodeElimination dce(callee_graph, stats_, "dead_code_elimination$inliner"); HConstantFolding fold(callee_graph, "constant_folding$inliner"); - HSharpening sharpening(callee_graph, codegen_, dex_compilation_unit, compiler_driver_); + HSharpening sharpening(callee_graph, codegen_, dex_compilation_unit, compiler_driver_, handles_); InstructionSimplifier simplify(callee_graph, stats_); IntrinsicsRecognizer intrinsics(callee_graph, stats_); diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index f4616e39e6..9d73e29602 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -64,7 +64,8 @@ HLoopOptimization::HLoopOptimization(HGraph* graph, top_loop_(nullptr), last_loop_(nullptr), iset_(nullptr), - induction_simplication_count_(0) { + induction_simplication_count_(0), + simplified_(false) { } void HLoopOptimization::Run() { @@ -169,9 +170,15 @@ void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { if (current_induction_simplification_count != induction_simplication_count_) { induction_range_.ReVisit(node->loop_info); } - SimplifyBlocks(node); - SimplifyInduction(node); - SimplifyBlocks(node); + // Repeat simplifications until no more changes occur. Note that since + // each simplification consists of eliminating code (without introducing + // new code), this process is always finite. + do { + simplified_ = false; + SimplifyBlocks(node); + SimplifyInduction(node); + } while (simplified_); + // Remove inner loops when empty. if (node->inner == nullptr) { RemoveIfEmptyInnerLoop(node); } @@ -198,63 +205,57 @@ void HLoopOptimization::SimplifyInduction(LoopNode* node) { for (HInstruction* i : *iset_) { RemoveFromCycle(i); } + simplified_ = true; induction_simplication_count_++; } } } void HLoopOptimization::SimplifyBlocks(LoopNode* node) { - // Repeat the block simplifications until no more changes occur. Note that since - // each simplification consists of eliminating code (without introducing new code), - // this process is always finite. - bool changed; - do { - changed = false; - // Iterate over all basic blocks in the loop-body. - for (HBlocksInLoopIterator it(*node->loop_info); !it.Done(); it.Advance()) { - HBasicBlock* block = it.Current(); - // Remove dead instructions from the loop-body. - for (HBackwardInstructionIterator i(block->GetInstructions()); !i.Done(); i.Advance()) { - HInstruction* instruction = i.Current(); - if (instruction->IsDeadAndRemovable()) { - changed = true; - block->RemoveInstruction(instruction); - } + // Iterate over all basic blocks in the loop-body. + for (HBlocksInLoopIterator it(*node->loop_info); !it.Done(); it.Advance()) { + HBasicBlock* block = it.Current(); + // Remove dead instructions from the loop-body. + for (HBackwardInstructionIterator i(block->GetInstructions()); !i.Done(); i.Advance()) { + HInstruction* instruction = i.Current(); + if (instruction->IsDeadAndRemovable()) { + simplified_ = true; + block->RemoveInstruction(instruction); } - // Remove trivial control flow blocks from the loop-body. - HBasicBlock* succ = nullptr; - if (IsGotoBlock(block, &succ) && succ->GetPredecessors().size() == 1) { - // Trivial goto block can be removed. - HBasicBlock* pred = block->GetSinglePredecessor(); - changed = true; - pred->ReplaceSuccessor(block, succ); - block->RemoveDominatedBlock(succ); - block->DisconnectAndDelete(); - pred->AddDominatedBlock(succ); - succ->SetDominator(pred); - } else if (block->GetSuccessors().size() == 2) { - // Trivial if block can be bypassed to either branch. - HBasicBlock* succ0 = block->GetSuccessors()[0]; - HBasicBlock* succ1 = block->GetSuccessors()[1]; - HBasicBlock* meet0 = nullptr; - HBasicBlock* meet1 = nullptr; - if (succ0 != succ1 && - IsGotoBlock(succ0, &meet0) && - IsGotoBlock(succ1, &meet1) && - meet0 == meet1 && // meets again - meet0 != block && // no self-loop - meet0->GetPhis().IsEmpty()) { // not used for merging - changed = true; - succ0->DisconnectAndDelete(); - if (block->Dominates(meet0)) { - block->RemoveDominatedBlock(meet0); - succ1->AddDominatedBlock(meet0); - meet0->SetDominator(succ1); - } + } + // Remove trivial control flow blocks from the loop-body. + HBasicBlock* succ = nullptr; + if (IsGotoBlock(block, &succ) && succ->GetPredecessors().size() == 1) { + // Trivial goto block can be removed. + HBasicBlock* pred = block->GetSinglePredecessor(); + simplified_ = true; + pred->ReplaceSuccessor(block, succ); + block->RemoveDominatedBlock(succ); + block->DisconnectAndDelete(); + pred->AddDominatedBlock(succ); + succ->SetDominator(pred); + } else if (block->GetSuccessors().size() == 2) { + // Trivial if block can be bypassed to either branch. + HBasicBlock* succ0 = block->GetSuccessors()[0]; + HBasicBlock* succ1 = block->GetSuccessors()[1]; + HBasicBlock* meet0 = nullptr; + HBasicBlock* meet1 = nullptr; + if (succ0 != succ1 && + IsGotoBlock(succ0, &meet0) && + IsGotoBlock(succ1, &meet1) && + meet0 == meet1 && // meets again + meet0 != block && // no self-loop + meet0->GetPhis().IsEmpty()) { // not used for merging + simplified_ = true; + succ0->DisconnectAndDelete(); + if (block->Dominates(meet0)) { + block->RemoveDominatedBlock(meet0); + succ1->AddDominatedBlock(meet0); + meet0->SetDominator(succ1); } } } - } while (changed); + } } void HLoopOptimization::RemoveIfEmptyInnerLoop(LoopNode* node) { diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h index 3391bef4e9..0f05b24c37 100644 --- a/compiler/optimizing/loop_optimization.h +++ b/compiler/optimizing/loop_optimization.h @@ -95,6 +95,9 @@ class HLoopOptimization : public HOptimization { // when the induction of inner loops has changed. int32_t induction_simplication_count_; + // Flag that tracks if any simplifications have occurred. + bool simplified_; + friend class LoopOptimizationTest; DISALLOW_COPY_AND_ASSIGN(HLoopOptimization); diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 594255c625..925d4f1fd1 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -2487,8 +2487,8 @@ std::ostream& operator<<(std::ostream& os, HLoadClass::LoadKind rhs) { return os << "BootImageLinkTimePcRelative"; case HLoadClass::LoadKind::kBootImageAddress: return os << "BootImageAddress"; - case HLoadClass::LoadKind::kDexCacheAddress: - return os << "DexCacheAddress"; + case HLoadClass::LoadKind::kJitTableAddress: + return os << "JitTableAddress"; case HLoadClass::LoadKind::kDexCachePcRelative: return os << "DexCachePcRelative"; case HLoadClass::LoadKind::kDexCacheViaMethod: diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index e3f4d8f035..659cddae05 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -5493,9 +5493,8 @@ class HLoadClass FINAL : public HInstruction { // GetIncludePatchInformation(). kBootImageAddress, - // Load from the resolved types array at an absolute address. - // Used for classes outside the boot image referenced by JIT-compiled code. - kDexCacheAddress, + // Load from the root table associated with the JIT compiled method. + kJitTableAddress, // Load from resolved types array in the dex cache using a PC-relative load. // Used for classes outside boot image when we know that we can access @@ -5588,7 +5587,6 @@ class HLoadClass FINAL : public HInstruction { NeedsAccessCheck(); } - bool CanThrow() const OVERRIDE { return CanCallRuntime(); } @@ -5613,7 +5611,9 @@ class HLoadClass FINAL : public HInstruction { return load_data_.address; } - bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { return !IsReferrersClass(); } + bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { + return !IsReferrersClass(); + } static SideEffects SideEffectsForArchRuntimeCalls() { return SideEffects::CanTriggerGC(); @@ -5672,7 +5672,8 @@ class HLoadClass FINAL : public HInstruction { } static bool HasAddress(LoadKind load_kind) { - return load_kind == LoadKind::kBootImageAddress || load_kind == LoadKind::kDexCacheAddress; + return load_kind == LoadKind::kBootImageAddress || + load_kind == LoadKind::kJitTableAddress; } static bool HasDexCacheReference(LoadKind load_kind) { @@ -5691,7 +5692,7 @@ class HLoadClass FINAL : public HInstruction { union { uint32_t dex_cache_element_index; // Only for dex cache reference. - uint64_t address; // Up to 64-bit, needed for kDexCacheAddress on 64-bit targets. + uint64_t address; // Up to 64-bit, needed for kJitTableAddress on 64-bit targets. } load_data_; ReferenceTypeInfo loaded_class_rti_; diff --git a/compiler/optimizing/optimizing_cfi_test.cc b/compiler/optimizing/optimizing_cfi_test.cc index 013e110b87..0e02311672 100644 --- a/compiler/optimizing/optimizing_cfi_test.cc +++ b/compiler/optimizing/optimizing_cfi_test.cc @@ -24,12 +24,22 @@ #include "optimizing/code_generator.h" #include "optimizing/optimizing_unit_test.h" #include "utils/assembler.h" +#ifdef ART_USE_VIXL_ARM_BACKEND +#include "utils/arm/assembler_arm_vixl.h" +#else #include "utils/arm/assembler_thumb2.h" +#endif #include "utils/mips/assembler_mips.h" #include "utils/mips64/assembler_mips64.h" #include "optimizing/optimizing_cfi_test_expected.inc" +#ifdef ART_USE_VIXL_ARM_BACKEND +namespace vixl32 = vixl::aarch32; + +using vixl32::r0; +#endif + namespace art { // Run the tests only on host. @@ -158,8 +168,7 @@ class OptimizingCFITest : public CFITest { TestImpl(isa, #isa, expected_asm, expected_cfi); \ } -// TODO(VIXL): Support this test for the VIXL backend. -#if defined(ART_ENABLE_CODEGEN_arm) && !defined(ART_USE_VIXL_ARM_BACKEND) +#ifdef ART_ENABLE_CODEGEN_arm TEST_ISA(kThumb2) #endif #ifdef ART_ENABLE_CODEGEN_arm64 @@ -178,8 +187,7 @@ TEST_ISA(kMips) TEST_ISA(kMips64) #endif -// TODO(VIXL): Support this test for the VIXL backend. -#if defined(ART_ENABLE_CODEGEN_arm) && !defined(ART_USE_VIXL_ARM_BACKEND) +#ifdef ART_ENABLE_CODEGEN_arm TEST_F(OptimizingCFITest, kThumb2Adjust) { std::vector<uint8_t> expected_asm( expected_asm_kThumb2_adjust, @@ -188,6 +196,16 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { expected_cfi_kThumb2_adjust, expected_cfi_kThumb2_adjust + arraysize(expected_cfi_kThumb2_adjust)); SetUpFrame(kThumb2); +#ifdef ART_USE_VIXL_ARM_BACKEND +#define __ down_cast<arm::ArmVIXLAssembler*>(GetCodeGenerator() \ + ->GetAssembler())->GetVIXLAssembler()-> + vixl32::Label target; + __ CompareAndBranchIfZero(r0, &target); + // Push the target out of range of CBZ. + for (size_t i = 0; i != 65; ++i) { + __ Ldr(r0, vixl32::MemOperand(r0)); + } +#else #define __ down_cast<arm::Thumb2Assembler*>(GetCodeGenerator()->GetAssembler())-> Label target; __ CompareAndBranchIfZero(arm::R0, &target); @@ -195,6 +213,7 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { for (size_t i = 0; i != 65; ++i) { __ ldr(arm::R0, arm::Address(arm::R0)); } +#endif __ Bind(&target); #undef __ Finish(); diff --git a/compiler/optimizing/optimizing_cfi_test_expected.inc b/compiler/optimizing/optimizing_cfi_test_expected.inc index f735dc8cb3..82670c38fe 100644 --- a/compiler/optimizing/optimizing_cfi_test_expected.inc +++ b/compiler/optimizing/optimizing_cfi_test_expected.inc @@ -223,8 +223,16 @@ static constexpr uint8_t expected_cfi_kMips64[] = { // 0x00000040: .cfi_def_cfa_offset: 64 static constexpr uint8_t expected_asm_kThumb2_adjust[] = { +#ifdef ART_USE_VIXL_ARM_BACKEND + // VIXL emits an extra 2 bytes here for a 32-bit beq as there is no + // optimistic 16-bit emit and subsequent fixup for out of reach targets + // as with the current assembler. + 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, 0x00, 0xF0, + 0x41, 0x80, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, +#else 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, 0x40, 0xD0, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, +#endif 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, @@ -239,7 +247,11 @@ static constexpr uint8_t expected_asm_kThumb2_adjust[] = { }; static constexpr uint8_t expected_cfi_kThumb2_adjust[] = { 0x42, 0x0E, 0x0C, 0x85, 0x03, 0x86, 0x02, 0x8E, 0x01, 0x44, 0x0E, 0x14, +#ifdef ART_USE_VIXL_ARM_BACKEND + 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x88, 0x0A, +#else 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x86, 0x0A, +#endif 0x42, 0x0E, 0x14, 0x44, 0x0E, 0x0C, 0x06, 0x50, 0x06, 0x51, 0x42, 0x0B, 0x0E, 0x40, }; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 8ea2b06530..64c87dc13a 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -375,7 +375,8 @@ class OptimizingCompiler FINAL : public Compiler { const DexFile& dex_file, Handle<mirror::DexCache> dex_cache, ArtMethod* method, - bool osr) const; + bool osr, + VariableSizedHandleScope* handles) const; void MaybeRunInliner(HGraph* graph, CodeGenerator* codegen, @@ -495,7 +496,7 @@ static HOptimization* BuildOptimization( number_of_dex_registers, /* depth */ 0); } else if (opt_name == HSharpening::kSharpeningPassName) { - return new (arena) HSharpening(graph, codegen, dex_compilation_unit, driver); + return new (arena) HSharpening(graph, codegen, dex_compilation_unit, driver, handles); } else if (opt_name == HSelectGenerator::kSelectGeneratorPassName) { return new (arena) HSelectGenerator(graph, stats); } else if (opt_name == HInductionVarAnalysis::kInductionPassName) { @@ -767,7 +768,8 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, HInductionVarAnalysis* induction = new (arena) HInductionVarAnalysis(graph); BoundsCheckElimination* bce = new (arena) BoundsCheckElimination(graph, *side_effects, induction); HLoopOptimization* loop = new (arena) HLoopOptimization(graph, induction); - HSharpening* sharpening = new (arena) HSharpening(graph, codegen, dex_compilation_unit, driver); + HSharpening* sharpening = new (arena) HSharpening( + graph, codegen, dex_compilation_unit, driver, handles); InstructionSimplifier* simplify2 = new (arena) InstructionSimplifier( graph, stats, "instruction_simplifier$after_inlining"); InstructionSimplifier* simplify3 = new (arena) InstructionSimplifier( @@ -866,7 +868,8 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, const DexFile& dex_file, Handle<mirror::DexCache> dex_cache, ArtMethod* method, - bool osr) const { + bool osr, + VariableSizedHandleScope* handles) const { MaybeRecordStat(MethodCompilationStat::kAttemptCompilation); CompilerDriver* compiler_driver = GetCompilerDriver(); InstructionSet instruction_set = compiler_driver->GetInstructionSet(); @@ -976,63 +979,55 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, compiler_driver, dump_mutex_); - VLOG(compiler) << "Building " << pass_observer.GetMethodName(); - { - ScopedObjectAccess soa(Thread::Current()); - VariableSizedHandleScope handles(soa.Self()); - // Do not hold `mutator_lock_` between optimizations. - ScopedThreadSuspension sts(soa.Self(), kNative); - - { - PassScope scope(HGraphBuilder::kBuilderPassName, &pass_observer); - HGraphBuilder builder(graph, - &dex_compilation_unit, - &dex_compilation_unit, - &dex_file, - *code_item, - compiler_driver, - compilation_stats_.get(), - interpreter_metadata, - dex_cache, - &handles); - GraphAnalysisResult result = builder.BuildGraph(); - if (result != kAnalysisSuccess) { - switch (result) { - case kAnalysisSkipped: - MaybeRecordStat(MethodCompilationStat::kNotCompiledSkipped); - break; - case kAnalysisInvalidBytecode: - MaybeRecordStat(MethodCompilationStat::kNotCompiledInvalidBytecode); - break; - case kAnalysisFailThrowCatchLoop: - MaybeRecordStat(MethodCompilationStat::kNotCompiledThrowCatchLoop); - break; - case kAnalysisFailAmbiguousArrayOp: - MaybeRecordStat(MethodCompilationStat::kNotCompiledAmbiguousArrayOp); - break; - case kAnalysisSuccess: - UNREACHABLE(); - } - pass_observer.SetGraphInBadState(); - return nullptr; + VLOG(compiler) << "Building " << pass_observer.GetMethodName(); + PassScope scope(HGraphBuilder::kBuilderPassName, &pass_observer); + HGraphBuilder builder(graph, + &dex_compilation_unit, + &dex_compilation_unit, + &dex_file, + *code_item, + compiler_driver, + compilation_stats_.get(), + interpreter_metadata, + dex_cache, + handles); + GraphAnalysisResult result = builder.BuildGraph(); + if (result != kAnalysisSuccess) { + switch (result) { + case kAnalysisSkipped: + MaybeRecordStat(MethodCompilationStat::kNotCompiledSkipped); + break; + case kAnalysisInvalidBytecode: + MaybeRecordStat(MethodCompilationStat::kNotCompiledInvalidBytecode); + break; + case kAnalysisFailThrowCatchLoop: + MaybeRecordStat(MethodCompilationStat::kNotCompiledThrowCatchLoop); + break; + case kAnalysisFailAmbiguousArrayOp: + MaybeRecordStat(MethodCompilationStat::kNotCompiledAmbiguousArrayOp); + break; + case kAnalysisSuccess: + UNREACHABLE(); } + pass_observer.SetGraphInBadState(); + return nullptr; } + } - RunOptimizations(graph, - codegen.get(), - compiler_driver, - dex_compilation_unit, - &pass_observer, - &handles); + RunOptimizations(graph, + codegen.get(), + compiler_driver, + dex_compilation_unit, + &pass_observer, + handles); - RegisterAllocator::Strategy regalloc_strategy = - compiler_options.GetRegisterAllocationStrategy(); - AllocateRegisters(graph, codegen.get(), &pass_observer, regalloc_strategy); + RegisterAllocator::Strategy regalloc_strategy = + compiler_options.GetRegisterAllocationStrategy(); + AllocateRegisters(graph, codegen.get(), &pass_observer, regalloc_strategy); - codegen->Compile(code_allocator); - pass_observer.DumpDisassembly(); - } + codegen->Compile(code_allocator); + pass_observer.DumpDisassembly(); return codegen.release(); } @@ -1055,19 +1050,27 @@ CompiledMethod* OptimizingCompiler::Compile(const DexFile::CodeItem* code_item, verified_method->GetEncounteredVerificationFailures())) { ArenaAllocator arena(Runtime::Current()->GetArenaPool()); CodeVectorAllocator code_allocator(&arena); - std::unique_ptr<CodeGenerator> codegen( - TryCompile(&arena, - &code_allocator, - code_item, - access_flags, - invoke_type, - class_def_idx, - method_idx, - jclass_loader, - dex_file, - dex_cache, - nullptr, - /* osr */ false)); + std::unique_ptr<CodeGenerator> codegen; + { + ScopedObjectAccess soa(Thread::Current()); + VariableSizedHandleScope handles(soa.Self()); + // Go to native so that we don't block GC during compilation. + ScopedThreadSuspension sts(soa.Self(), kNative); + codegen.reset( + TryCompile(&arena, + &code_allocator, + code_item, + access_flags, + invoke_type, + class_def_idx, + method_idx, + jclass_loader, + dex_file, + dex_cache, + nullptr, + /* osr */ false, + &handles)); + } if (codegen.get() != nullptr) { MaybeRecordStat(MethodCompilationStat::kCompiled); method = Emit(&arena, &code_allocator, codegen.get(), compiler_driver, code_item); @@ -1138,6 +1141,8 @@ bool OptimizingCompiler::JitCompile(Thread* self, ArenaAllocator arena(Runtime::Current()->GetJitArenaPool()); CodeVectorAllocator code_allocator(&arena); + VariableSizedHandleScope handles(self); + std::unique_ptr<CodeGenerator> codegen; { // Go to native so that we don't block GC during compilation. @@ -1154,7 +1159,8 @@ bool OptimizingCompiler::JitCompile(Thread* self, *dex_file, dex_cache, method, - osr)); + osr, + &handles)); if (codegen.get() == nullptr) { return false; } diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index daf160a483..bbbb1a18ce 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -151,7 +151,7 @@ void HSharpening::ProcessLoadClass(HLoadClass* load_class) { bool is_in_dex_cache = false; bool is_in_boot_image = false; - HLoadClass::LoadKind desired_load_kind; + HLoadClass::LoadKind desired_load_kind = static_cast<HLoadClass::LoadKind>(-1); uint64_t address = 0u; // Class or dex cache element address. { ScopedObjectAccess soa(Thread::Current()); @@ -190,18 +190,19 @@ void HSharpening::ProcessLoadClass(HLoadClass* load_class) { // TODO: Use direct pointers for all non-moving spaces, not just boot image. Bug: 29530787 desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; address = reinterpret_cast64<uint64_t>(klass); + } else if (is_in_dex_cache) { + desired_load_kind = HLoadClass::LoadKind::kJitTableAddress; + // We store in the address field the location of the stack reference maintained + // by the handle. We do this now so that the code generation does not need to figure + // out which class loader to use. + address = reinterpret_cast<uint64_t>(handles_->NewHandle(klass).GetReference()); } else { - // Note: If the class is not in the dex cache or isn't initialized, the - // instruction needs environment and will not be inlined across dex files. - // Within a dex file, the slow-path helper loads the correct class and - // inlined frames are used correctly for OOM stack trace. - // TODO: Write a test for this. Bug: 29416588 - desired_load_kind = HLoadClass::LoadKind::kDexCacheAddress; - void* dex_cache_element_address = &dex_cache->GetResolvedTypes()[type_index.index_]; - address = reinterpret_cast64<uint64_t>(dex_cache_element_address); + // Class not loaded yet. Fallback to the dex cache. + // TODO(ngeoffray): Generate HDeoptimize instead. + desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; } - // AOT app compilation. Check if the class is in the boot image. } else if (is_in_boot_image && !codegen_->GetCompilerOptions().GetCompilePic()) { + // AOT app compilation. Check if the class is in the boot image. desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; address = reinterpret_cast64<uint64_t>(klass); } else { @@ -215,6 +216,7 @@ void HSharpening::ProcessLoadClass(HLoadClass* load_class) { } } } + DCHECK_NE(desired_load_kind, static_cast<HLoadClass::LoadKind>(-1)); if (is_in_boot_image) { load_class->MarkInBootImage(); @@ -245,7 +247,7 @@ void HSharpening::ProcessLoadClass(HLoadClass* load_class) { load_class->SetLoadKindWithTypeReference(load_kind, dex_file, type_index); break; case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK_NE(address, 0u); load_class->SetLoadKindWithAddress(load_kind, address); break; diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index d35ae66e05..74189549fd 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -35,11 +35,13 @@ class HSharpening : public HOptimization { HSharpening(HGraph* graph, CodeGenerator* codegen, const DexCompilationUnit& compilation_unit, - CompilerDriver* compiler_driver) + CompilerDriver* compiler_driver, + VariableSizedHandleScope* handles) : HOptimization(graph, kSharpeningPassName), codegen_(codegen), compilation_unit_(compilation_unit), - compiler_driver_(compiler_driver) { } + compiler_driver_(compiler_driver), + handles_(handles) { } void Run() OVERRIDE; @@ -53,6 +55,7 @@ class HSharpening : public HOptimization { CodeGenerator* codegen_; const DexCompilationUnit& compilation_unit_; CompilerDriver* compiler_driver_; + VariableSizedHandleScope* handles_; }; } // namespace art |