diff options
21 files changed, 504 insertions, 77 deletions
diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index 8b450e11dc..9f6b78a82c 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -1375,4 +1375,33 @@ uint32_t CodeGenerator::GetReferenceDisableFlagOffset() const { return klass->GetDisableIntrinsicFlagOffset().Uint32Value(); } +void CodeGenerator::EmitJitRoots(uint8_t* code, + Handle<mirror::ObjectArray<mirror::Object>> roots, + const uint8_t* roots_data, + Handle<mirror::DexCache> outer_dex_cache) { + DCHECK_EQ(static_cast<size_t>(roots->GetLength()), GetNumberOfJitRoots()); + StackHandleScope<1> hs(Thread::Current()); + MutableHandle<mirror::DexCache> h_dex_cache(hs.NewHandle<mirror::DexCache>(nullptr)); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + size_t index = 0; + for (auto& entry : jit_string_roots_) { + const DexFile& entry_dex_file = *entry.first.dex_file; + // Avoid the expensive FindDexCache call by checking if the string is + // in the compiled method's dex file. + h_dex_cache.Assign(IsSameDexFile(*outer_dex_cache->GetDexFile(), entry_dex_file) + ? outer_dex_cache.Get() + : class_linker->FindDexCache(hs.Self(), entry_dex_file)); + mirror::String* string = class_linker->LookupString( + entry_dex_file, entry.first.string_index, h_dex_cache); + DCHECK(string != nullptr) << "JIT roots require strings to have been loaded"; + // Ensure the string is strongly interned. This is a requirement on how the JIT + // handles strings. b/32995596 + class_linker->GetInternTable()->InternStrong(string); + roots->Set(index, string); + entry.second = index; + ++index; + } + EmitJitRootPatches(code, roots_data); +} + } // namespace art diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index bf246ad309..a5d19abe92 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -32,6 +32,7 @@ #include "optimizing_compiler_stats.h" #include "read_barrier_option.h" #include "stack_map_stream.h" +#include "string_reference.h" #include "utils/label.h" namespace art { @@ -335,6 +336,17 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { void BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item); size_t ComputeStackMapsSize(); + size_t GetNumberOfJitRoots() const { + return jit_string_roots_.size(); + } + + // Fills the `literals` array with literals collected during code generation. + // Also emits literal patches. + void EmitJitRoots(uint8_t* code, + Handle<mirror::ObjectArray<mirror::Object>> roots, + const uint8_t* roots_data, + Handle<mirror::DexCache> outer_dex_cache) + REQUIRES_SHARED(Locks::mutator_lock_); bool IsLeafMethod() const { return is_leaf_; @@ -515,6 +527,26 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { virtual HLoadClass::LoadKind GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) = 0; + static LocationSummary::CallKind GetLoadStringCallKind(HLoadString* load) { + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBssEntry: + DCHECK(load->NeedsEnvironment()); + return LocationSummary::kCallOnSlowPath; + case HLoadString::LoadKind::kDexCacheViaMethod: + DCHECK(load->NeedsEnvironment()); + return LocationSummary::kCallOnMainOnly; + case HLoadString::LoadKind::kJitTableAddress: + DCHECK(!load->NeedsEnvironment()); + return kEmitCompilerReadBarrier + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; + break; + default: + DCHECK(!load->NeedsEnvironment()); + return LocationSummary::kNoCall; + } + } + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. virtual HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -571,6 +603,8 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { fpu_callee_save_mask_(fpu_callee_save_mask), stack_map_stream_(graph->GetArena()), block_order_(nullptr), + jit_string_roots_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), disasm_info_(nullptr), stats_(stats), graph_(graph), @@ -637,6 +671,12 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { return current_slow_path_; } + // Emit the patches assocatied with JIT roots. Only applies to JIT compiled code. + virtual void EmitJitRootPatches(uint8_t* code ATTRIBUTE_UNUSED, + const uint8_t* roots_data ATTRIBUTE_UNUSED) { + DCHECK_EQ(jit_string_roots_.size(), 0u); + } + // Frame size required for this method. uint32_t frame_size_; uint32_t core_spill_mask_; @@ -662,6 +702,11 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { // The order to use for code generation. const ArenaVector<HBasicBlock*>* block_order_; + // Maps a StringReference (dex_file, string_index) to the index in the literal table. + // Entries are intially added with a 0 index, and `EmitJitRoots` will compute all the + // indices. + ArenaSafeMap<StringReference, size_t, StringReferenceValueComparator> jit_string_roots_; + DisassemblyInformation* disasm_info_; private: diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 7c3a2c6b6e..a341086ac4 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -1214,7 +1214,9 @@ CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_address_patches_(std::less<uint32_t>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Always save the LR register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(LR)); } @@ -5893,6 +5895,9 @@ HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; + case HLoadString::LoadKind::kJitTableAddress: + DCHECK(Runtime::Current()->UseJitCompilation()); + break; case HLoadString::LoadKind::kDexCacheViaMethod: break; } @@ -5900,13 +5905,8 @@ HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( } void LocationsBuilderARM::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->NeedsEnvironment() - ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) - ? LocationSummary::kCallOnMainOnly - : LocationSummary::kCallOnSlowPath) - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - HLoadString::LoadKind load_kind = load->GetLoadKind(); if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { locations->SetOut(Location::RegisterLocation(R0)); @@ -5979,6 +5979,13 @@ void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: { + __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + // /* GcRoot<mirror::String> */ out = *out + GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + return; + } default: break; } @@ -7356,6 +7363,14 @@ Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { return DeduplicateUint32Literal(address, &uint32_literals_); } +Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, + uint32_t string_index) { + jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); + return jit_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( const ArenaDeque<PcRelativePatchInfo>& infos, @@ -7672,6 +7687,21 @@ void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction } } +void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const auto& entry : jit_string_patches_) { + const auto& it = jit_string_roots_.find(entry.first); + DCHECK(it != jit_string_roots_.end()); + size_t index_in_table = it->second; + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = literal->GetLabel()->Position(); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint8_t* data = code + literal_offset; + reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); + } +} + #undef __ #undef QUICK_ENTRY_POINT diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index f95dd573cb..8ace3dac08 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -488,9 +488,12 @@ class CodeGeneratorARM : public CodeGenerator { Literal* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, uint32_t type_index); Literal* DeduplicateBootImageAddressLiteral(uint32_t address); Literal* DeduplicateDexCacheAddressLiteral(uint32_t address); + Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, uint32_t string_index); void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, @@ -591,9 +594,9 @@ class CodeGeneratorARM : public CodeGenerator { using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>; - using BootStringToLiteralMap = ArenaSafeMap<StringReference, - Literal*, - StringReferenceValueComparator>; + using StringToLiteralMap = ArenaSafeMap<StringReference, + Literal*, + StringReferenceValueComparator>; using BootTypeToLiteralMap = ArenaSafeMap<TypeReference, Literal*, TypeReferenceValueComparator>; @@ -605,7 +608,6 @@ class CodeGeneratorARM : public CodeGenerator { PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches); - template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos, ArenaVector<LinkerPatch>* linker_patches); @@ -630,7 +632,7 @@ class CodeGeneratorARM : public CodeGenerator { // PC-relative patch info for each HArmDexCacheArraysBase. ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // Deduplication map for boot string literals for kBootImageLinkTimeAddress. - BootStringToLiteralMap boot_image_string_patches_; + StringToLiteralMap boot_image_string_patches_; // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; // Deduplication map for boot type literals for kBootImageLinkTimeAddress. @@ -640,6 +642,9 @@ class CodeGeneratorARM : public CodeGenerator { // Deduplication map for patchable boot image addresses. Uint32ToLiteralMap boot_image_address_patches_; + // Patches for string literals in JIT compiled code. + StringToLiteralMap jit_string_patches_; + DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM); }; diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 6ec9c910ec..c9e563b095 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -1158,7 +1158,9 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_address_patches_(std::less<uint32_t>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save the link register (containing the return address) to mimic Quick. AddAllocatedRegister(LocationFrom(lr)); } @@ -4152,6 +4154,14 @@ vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddress return DeduplicateUint64Literal(address); } +vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLiteral( + const DexFile& dex_file, uint32_t string_index) { + jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); + return jit_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); +} + void CodeGeneratorARM64::EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg) { DCHECK(reg.IsX()); @@ -4537,16 +4547,15 @@ HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind( break; case HLoadString::LoadKind::kDexCacheViaMethod: break; + case HLoadString::LoadKind::kJitTableAddress: + DCHECK(Runtime::Current()->UseJitCompilation()); + break; } return desired_string_load_kind; } void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->NeedsEnvironment() - ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) - ? LocationSummary::kCallOnMainOnly - : LocationSummary::kCallOnSlowPath) - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) { InvokeRuntimeCallingConvention calling_convention; @@ -4572,6 +4581,7 @@ void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) { Register out = OutputRegister(load); + Location out_loc = load->GetLocations()->Out(); switch (load->GetLoadKind()) { case HLoadString::LoadKind::kBootImageLinkTimeAddress: @@ -4608,9 +4618,9 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) { // Add LDR with its PC-relative String patch. vixl::aarch64::Label* ldr_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label); - // /* GcRoot<mirror::Class> */ out = *(base_address + offset) /* PC-relative */ + // /* GcRoot<mirror::String> */ out = *(base_address + offset) /* PC-relative */ GenerateGcRootFieldLoad(load, - load->GetLocations()->Out(), + out_loc, temp, /* offset placeholder */ 0u, ldr_label, @@ -4622,6 +4632,17 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) { __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: { + __ Ldr(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + GenerateGcRootFieldLoad(load, + out_loc, + out.X(), + /* offset */ 0, + /* fixup_label */ nullptr, + kCompilerReadBarrierOption); + return; + } default: break; } @@ -5741,7 +5762,19 @@ void InstructionCodeGeneratorARM64::VisitClassTableGet(HClassTableGet* instructi } } - +void CodeGeneratorARM64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const auto& entry : jit_string_patches_) { + const auto& it = jit_string_roots_.find(entry.first); + DCHECK(it != jit_string_roots_.end()); + size_t index_in_table = it->second; + vixl::aarch64::Literal<uint32_t>* literal = entry.second; + uint32_t literal_offset = literal->GetOffset(); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint8_t* data = code + literal_offset; + reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); + } +} #undef __ #undef QUICK_ENTRY_POINT diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 0e8d4fd549..a2ab60709f 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -565,6 +565,8 @@ class CodeGeneratorARM64 : public CodeGenerator { uint32_t type_index); vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address); vixl::aarch64::Literal<uint64_t>* DeduplicateDexCacheAddressLiteral(uint64_t address); + vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file, + uint32_t string_index); void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg); void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label, @@ -576,6 +578,8 @@ class CodeGeneratorARM64 : public CodeGenerator { void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, @@ -673,9 +677,9 @@ class CodeGeneratorARM64 : public CodeGenerator { using MethodToLiteralMap = ArenaSafeMap<MethodReference, vixl::aarch64::Literal<uint64_t>*, MethodReferenceComparator>; - using BootStringToLiteralMap = ArenaSafeMap<StringReference, - vixl::aarch64::Literal<uint32_t>*, - StringReferenceValueComparator>; + using StringToLiteralMap = ArenaSafeMap<StringReference, + vixl::aarch64::Literal<uint32_t>*, + StringReferenceValueComparator>; using BootTypeToLiteralMap = ArenaSafeMap<TypeReference, vixl::aarch64::Literal<uint32_t>*, TypeReferenceValueComparator>; @@ -739,7 +743,7 @@ class CodeGeneratorARM64 : public CodeGenerator { // PC-relative DexCache access info. ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // Deduplication map for boot string literals for kBootImageLinkTimeAddress. - BootStringToLiteralMap boot_image_string_patches_; + StringToLiteralMap boot_image_string_patches_; // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; // Deduplication map for boot type literals for kBootImageLinkTimeAddress. @@ -749,6 +753,9 @@ class CodeGeneratorARM64 : public CodeGenerator { // Deduplication map for patchable boot image addresses. Uint32ToLiteralMap boot_image_address_patches_; + // Patches for string literals in JIT compiled code. + StringToLiteralMap jit_string_patches_; + DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); }; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 573bb507f2..fcbb8f03ad 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -5207,6 +5207,11 @@ HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( case HLoadString::LoadKind::kDexCacheViaMethod: fallback_load = false; break; + case HLoadString::LoadKind::kJitTableAddress: + DCHECK(Runtime::Current()->UseJitCompilation()); + // TODO: implement. + fallback_load = true; + break; } if (fallback_load) { desired_string_load_kind = HLoadString::LoadKind::kDexCacheViaMethod; diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 7e4ad267c8..38494370ab 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -1012,6 +1012,7 @@ CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), constant_area_start_(-1), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), method_address_offset_(-1) { @@ -6218,16 +6219,15 @@ HLoadString::LoadKind CodeGeneratorX86::GetSupportedLoadStringKind( break; case HLoadString::LoadKind::kDexCacheViaMethod: break; + case HLoadString::LoadKind::kJitTableAddress: + DCHECK(Runtime::Current()->UseJitCompilation()); + break; } return desired_string_load_kind; } void LocationsBuilderX86::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) - ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) - ? LocationSummary::kCallOnMainOnly - : LocationSummary::kCallOnSlowPath) - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); HLoadString::LoadKind load_kind = load->GetLoadKind(); if (load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative || @@ -6252,6 +6252,14 @@ void LocationsBuilderX86::VisitLoadString(HLoadString* load) { } } +Label* CodeGeneratorX86::NewJitRootStringPatch(const DexFile& dex_file, uint32_t dex_index) { + jit_string_roots_.Overwrite(StringReference(&dex_file, dex_index), /* placeholder */ 0u); + // Add a patch entry and return the label. + jit_string_patches_.emplace_back(dex_file, dex_index); + PatchInfo<Label>* info = &jit_string_patches_.back(); + return &info->label; +} + void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); @@ -6280,7 +6288,7 @@ void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) { Register method_address = locations->InAt(0).AsRegister<Register>(); Address address = Address(method_address, CodeGeneratorX86::kDummy32BitOffset); Label* fixup_label = codegen_->NewStringBssEntryPatch(load); - // /* GcRoot<mirror::Class> */ out = *address /* PC-relative */ + // /* GcRoot<mirror::String> */ out = *address /* PC-relative */ GenerateGcRootFieldLoad(load, out_loc, address, fixup_label, kCompilerReadBarrierOption); SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load); codegen_->AddSlowPath(slow_path); @@ -6289,6 +6297,14 @@ void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) { __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: { + Address address = Address::Absolute(CodeGeneratorX86::kDummy32BitOffset); + Label* fixup_label = codegen_->NewJitRootStringPatch( + load->GetDexFile(), load->GetStringIndex()); + // /* GcRoot<mirror::String> */ out = *address + GenerateGcRootFieldLoad(load, out_loc, address, fixup_label, kCompilerReadBarrierOption); + return; + } default: break; } @@ -7736,6 +7752,20 @@ void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type t } } +void CodeGeneratorX86::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const PatchInfo<Label>& info : jit_string_patches_) { + const auto& it = jit_string_roots_.find(StringReference(&info.dex_file, info.index)); + DCHECK(it != jit_string_roots_.end()); + size_t index_in_table = it->second; + uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; + reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = + dchecked_integral_cast<uint32_t>(address); + } +} + #undef __ } // namespace x86 diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 164231b4e5..16ea6b55d6 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -414,12 +414,15 @@ class CodeGeneratorX86 : public CodeGenerator { void RecordTypePatch(HLoadClass* load_class); Label* NewStringBssEntryPatch(HLoadString* load_string); Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); + Label* NewJitRootStringPatch(const DexFile& dex_file, uint32_t dex_index); void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + // Emit a write barrier. void MarkGCCard(Register temp, Register card, @@ -616,6 +619,9 @@ class CodeGeneratorX86 : public CodeGenerator { // Type patch locations. ArenaDeque<PatchInfo<Label>> type_patches_; + // Patches for string root accesses in JIT compiled code. + ArenaDeque<PatchInfo<Label>> jit_string_patches_; + // Offset to the start of the constant area in the assembled code. // Used for fixups to the constant area. int32_t constant_area_start_; diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 19b30192cd..02f549151c 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -1258,7 +1258,8 @@ CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph, simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister)); } @@ -5630,16 +5631,15 @@ HLoadString::LoadKind CodeGeneratorX86_64::GetSupportedLoadStringKind( break; case HLoadString::LoadKind::kDexCacheViaMethod: break; + case HLoadString::LoadKind::kJitTableAddress: + DCHECK(Runtime::Current()->UseJitCompilation()); + break; } return desired_string_load_kind; } void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->NeedsEnvironment() - ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) - ? LocationSummary::kCallOnMainOnly - : LocationSummary::kCallOnSlowPath) - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) { locations->SetOut(Location::RegisterLocation(RAX)); @@ -5659,6 +5659,14 @@ void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) { } } +Label* CodeGeneratorX86_64::NewJitRootStringPatch(const DexFile& dex_file, uint32_t dex_index) { + jit_string_roots_.Overwrite(StringReference(&dex_file, dex_index), /* placeholder */ 0u); + // Add a patch entry and return the label. + jit_string_patches_.emplace_back(dex_file, dex_index); + PatchInfo<Label>* info = &jit_string_patches_.back(); + return &info->label; +} + void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); @@ -5690,6 +5698,15 @@ void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) { __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: { + Address address = Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, + /* no_rip */ true); + Label* fixup_label = + codegen_->NewJitRootStringPatch(load->GetDexFile(), load->GetStringIndex()); + // /* GcRoot<mirror::String> */ out = *address + GenerateGcRootFieldLoad(load, out_loc, address, fixup_label, kCompilerReadBarrierOption); + return; + } default: break; } @@ -7091,6 +7108,20 @@ void CodeGeneratorX86_64::MoveInt64ToAddress(const Address& addr_low, } } +void CodeGeneratorX86_64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const PatchInfo<Label>& info : jit_string_patches_) { + const auto& it = jit_string_roots_.find(StringReference(&info.dex_file, info.index)); + DCHECK(it != jit_string_roots_.end()); + size_t index_in_table = it->second; + uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; + reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = + dchecked_integral_cast<uint32_t>(address); + } +} + #undef __ } // namespace x86_64 diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index e5a4152517..0f70b15787 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -412,11 +412,14 @@ class CodeGeneratorX86_64 : public CodeGenerator { void RecordTypePatch(HLoadClass* load_class); Label* NewStringBssEntryPatch(HLoadString* load_string); Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); + Label* NewJitRootStringPatch(const DexFile& dex_file, uint32_t dex_index); void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + const X86_64InstructionSetFeatures& GetInstructionSetFeatures() const { return isa_features_; } @@ -602,6 +605,9 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Fixups for jump tables need to be handled specially. ArenaVector<JumpTableRIPFixup*> fixups_to_jump_tables_; + // Patches for string literals in JIT compiled code. + ArenaDeque<PatchInfo<Label>> jit_string_patches_; + DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86_64); }; diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index e0c582a76d..215ed54a4a 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -5690,7 +5690,10 @@ class HLoadString FINAL : public HInstruction { // all other types are unavailable. kDexCacheViaMethod, - kLast = kDexCacheViaMethod + // Load from the root table associated with the JIT compiled method. + kJitTableAddress, + + kLast = kJitTableAddress, }; HLoadString(HCurrentMethod* current_method, @@ -5748,7 +5751,8 @@ class HLoadString FINAL : public HInstruction { LoadKind load_kind = GetLoadKind(); if (load_kind == LoadKind::kBootImageLinkTimeAddress || load_kind == LoadKind::kBootImageLinkTimePcRelative || - load_kind == LoadKind::kBootImageAddress) { + load_kind == LoadKind::kBootImageAddress || + load_kind == LoadKind::kJitTableAddress) { return false; } return !IsInDexCache(); @@ -5801,7 +5805,8 @@ class HLoadString FINAL : public HInstruction { return load_kind == LoadKind::kBootImageLinkTimeAddress || load_kind == LoadKind::kBootImageLinkTimePcRelative || load_kind == LoadKind::kBssEntry || - load_kind == LoadKind::kDexCacheViaMethod; + load_kind == LoadKind::kDexCacheViaMethod || + load_kind == LoadKind::kJitTableAddress; } static bool HasAddress(LoadKind load_kind) { diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 499514de97..85519c96e4 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -117,6 +117,7 @@ class CodeVectorAllocator FINAL : public CodeAllocator { size_t GetSize() const { return size_; } const ArenaVector<uint8_t>& GetMemory() const { return memory_; } + uint8_t* GetData() { return memory_.data(); } private: ArenaVector<uint8_t> memory_; @@ -1124,7 +1125,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, jit::JitCodeCache* code_cache, ArtMethod* method, bool osr) { - StackHandleScope<2> hs(self); + StackHandleScope<3> hs(self); Handle<mirror::ClassLoader> class_loader(hs.NewHandle( method->GetDeclaringClass()->GetClassLoader())); Handle<mirror::DexCache> dex_cache(hs.NewHandle(method->GetDexCache())); @@ -1170,22 +1171,43 @@ bool OptimizingCompiler::JitCompile(Thread* self, } size_t stack_map_size = codegen->ComputeStackMapsSize(); - uint8_t* stack_map_data = code_cache->ReserveData(self, stack_map_size, method); - if (stack_map_data == nullptr) { + size_t number_of_roots = codegen->GetNumberOfJitRoots(); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + // We allocate an object array to ensure the JIT roots that we will collect in EmitJitRoots + // will be visible by the GC between EmitLiterals and CommitCode. Once CommitCode is + // executed, this array is not needed. + Handle<mirror::ObjectArray<mirror::Object>> roots( + hs.NewHandle(mirror::ObjectArray<mirror::Object>::Alloc( + self, class_linker->GetClassRoot(ClassLinker::kObjectArrayClass), number_of_roots))); + if (roots.Get() == nullptr) { + // Out of memory, just clear the exception to avoid any Java exception uncaught problems. + DCHECK(self->IsExceptionPending()); + self->ClearException(); + return false; + } + uint8_t* stack_map_data = nullptr; + uint8_t* roots_data = nullptr; + code_cache->ReserveData( + self, stack_map_size, number_of_roots, method, &stack_map_data, &roots_data); + if (stack_map_data == nullptr || roots_data == nullptr) { return false; } MaybeRecordStat(MethodCompilationStat::kCompiled); codegen->BuildStackMaps(MemoryRegion(stack_map_data, stack_map_size), *code_item); + codegen->EmitJitRoots(code_allocator.GetData(), roots, roots_data, dex_cache); + const void* code = code_cache->CommitCode( self, method, stack_map_data, + roots_data, codegen->HasEmptyFrame() ? 0 : codegen->GetFrameSize(), codegen->GetCoreSpillMask(), codegen->GetFpuSpillMask(), code_allocator.GetMemory().data(), code_allocator.GetSize(), - osr); + osr, + roots); if (code == nullptr) { code_cache->ClearData(self, stack_map_data); diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index 63e4ca674e..15254edcab 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -281,7 +281,8 @@ void HSharpening::ProcessLoadString(HLoadString* load_string) { : hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file)); if (codegen_->GetCompilerOptions().IsBootImage()) { - // Compiling boot image. Resolve the string and allocate it if needed. + // Compiling boot image. Resolve the string and allocate it if needed, to ensure + // the string will be added to the boot image. DCHECK(!runtime->UseJitCompilation()); mirror::String* string = class_linker->ResolveString(dex_file, string_index, dex_cache); CHECK(string != nullptr); @@ -297,10 +298,14 @@ void HSharpening::ProcessLoadString(HLoadString* load_string) { } else if (runtime->UseJitCompilation()) { // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); - mirror::String* string = dex_cache->GetResolvedString(string_index); - if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) { - desired_load_kind = HLoadString::LoadKind::kBootImageAddress; - address = reinterpret_cast64<uint64_t>(string); + mirror::String* string = class_linker->LookupString(dex_file, string_index, dex_cache); + if (string != nullptr) { + if (runtime->GetHeap()->ObjectIsInBootImageSpace(string)) { + desired_load_kind = HLoadString::LoadKind::kBootImageAddress; + address = reinterpret_cast64<uint64_t>(string); + } else { + desired_load_kind = HLoadString::LoadKind::kJitTableAddress; + } } } else { // AOT app compilation. Try to lookup the string without allocating if not found. @@ -322,6 +327,7 @@ void HSharpening::ProcessLoadString(HLoadString* load_string) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: case HLoadString::LoadKind::kBssEntry: case HLoadString::LoadKind::kDexCacheViaMethod: + case HLoadString::LoadKind::kJitTableAddress: load_string->SetLoadKindWithStringReference(load_kind, dex_file, string_index); break; case HLoadString::LoadKind::kBootImageAddress: diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index a26d8502a3..719faedc84 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -80,8 +80,18 @@ JitCodeCache* JitCodeCache::Create(size_t initial_capacity, std::string error_str; // Map name specific for android_os_Debug.cpp accounting. + // Map in low 4gb to simplify accessing root tables for x86_64. + // We could do PC-relative addressing to avoid this problem, but that + // would require reserving code and data area before submitting, which + // means more windows for the code memory to be RWX. MemMap* data_map = MemMap::MapAnonymous( - "data-code-cache", nullptr, max_capacity, kProtAll, false, false, &error_str, use_ashmem); + "data-code-cache", nullptr, + max_capacity, + kProtAll, + /* low_4gb */ true, + /* reuse */ false, + &error_str, + use_ashmem); if (data_map == nullptr) { std::ostringstream oss; oss << "Failed to create read write execute cache: " << error_str << " size=" << max_capacity; @@ -197,34 +207,40 @@ class ScopedCodeCacheWrite : ScopedTrace { uint8_t* JitCodeCache::CommitCode(Thread* self, ArtMethod* method, - const uint8_t* vmap_table, + uint8_t* stack_map, + uint8_t* roots_data, size_t frame_size_in_bytes, size_t core_spill_mask, size_t fp_spill_mask, const uint8_t* code, size_t code_size, - bool osr) { + bool osr, + Handle<mirror::ObjectArray<mirror::Object>> roots) { uint8_t* result = CommitCodeInternal(self, method, - vmap_table, + stack_map, + roots_data, frame_size_in_bytes, core_spill_mask, fp_spill_mask, code, code_size, - osr); + osr, + roots); if (result == nullptr) { // Retry. GarbageCollectCache(self); result = CommitCodeInternal(self, method, - vmap_table, + stack_map, + roots_data, frame_size_in_bytes, core_spill_mask, fp_spill_mask, code, code_size, - osr); + osr, + roots); } return result; } @@ -243,20 +259,78 @@ static uintptr_t FromCodeToAllocation(const void* code) { return reinterpret_cast<uintptr_t>(code) - RoundUp(sizeof(OatQuickMethodHeader), alignment); } +static uint32_t ComputeRootTableSize(uint32_t number_of_roots) { + return sizeof(uint32_t) + number_of_roots * sizeof(GcRoot<mirror::Object>); +} + +static uint32_t GetNumberOfRoots(const uint8_t* stack_map) { + // The length of the table is stored just before the stack map (and therefore at the end of + // the table itself), in order to be able to fetch it from a `stack_map` pointer. + return reinterpret_cast<const uint32_t*>(stack_map)[-1]; +} + +static void FillRootTable(uint8_t* roots_data, Handle<mirror::ObjectArray<mirror::Object>> roots) + REQUIRES_SHARED(Locks::mutator_lock_) { + GcRoot<mirror::Object>* gc_roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data); + uint32_t length = roots->GetLength(); + // Put all roots in `roots_data`. + for (uint32_t i = 0; i < length; ++i) { + ObjPtr<mirror::Object> object = roots->Get(i); + if (kIsDebugBuild) { + // Ensure the string is strongly interned. b/32995596 + CHECK(object->IsString()); + ObjPtr<mirror::String> str = reinterpret_cast<mirror::String*>(object.Ptr()); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + CHECK(class_linker->GetInternTable()->LookupStrong(Thread::Current(), str) != nullptr); + } + gc_roots[i] = GcRoot<mirror::Object>(object); + } + // Store the length of the table at the end. This will allow fetching it from a `stack_map` + // pointer. + reinterpret_cast<uint32_t*>(gc_roots + length)[0] = length; +} + +static uint8_t* GetRootTable(const void* code_ptr, uint32_t* number_of_roots = nullptr) { + OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); + uint8_t* data = method_header->GetOptimizedCodeInfoPtr(); + uint32_t roots = GetNumberOfRoots(data); + if (number_of_roots != nullptr) { + *number_of_roots = roots; + } + return data - ComputeRootTableSize(roots); +} + +void JitCodeCache::SweepRootTables(IsMarkedVisitor* visitor) { + MutexLock mu(Thread::Current(), lock_); + for (const auto& entry : method_code_map_) { + uint32_t number_of_roots = 0; + uint8_t* roots_data = GetRootTable(entry.first, &number_of_roots); + GcRoot<mirror::Object>* roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data); + for (uint32_t i = 0; i < number_of_roots; ++i) { + // This does not need a read barrier because this is called by GC. + mirror::Object* object = roots[i].Read<kWithoutReadBarrier>(); + DCHECK(object != nullptr); + mirror::Object* new_object = visitor->IsMarked(object); + // We know the string is marked because it's a strongly-interned string that + // is always alive. The IsMarked implementation of the CMS collector returns + // null for newly allocated objects, but we know those haven't moved. Therefore, + // only update the entry if we get a different non-null string. + // TODO: Do not use IsMarked for j.l.Class, and adjust once we move this method + // out of the weak access/creation pause. b/32167580 + if (new_object != nullptr && new_object != object) { + DCHECK(new_object->IsString()); + roots[i] = GcRoot<mirror::Object>(new_object); + } + } + } +} + void JitCodeCache::FreeCode(const void* code_ptr, ArtMethod* method ATTRIBUTE_UNUSED) { uintptr_t allocation = FromCodeToAllocation(code_ptr); - const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); // Notify native debugger that we are about to remove the code. // It does nothing if we are not using native debugger. DeleteJITCodeEntryForAddress(reinterpret_cast<uintptr_t>(code_ptr)); - - // Use the offset directly to prevent sanity check that the method is - // compiled with optimizing. - // TODO(ngeoffray): Clean up. - if (method_header->vmap_table_offset_ != 0) { - const uint8_t* data = method_header->code_ - method_header->vmap_table_offset_; - FreeData(const_cast<uint8_t*>(data)); - } + FreeData(GetRootTable(code_ptr)); FreeCode(reinterpret_cast<uint8_t*>(allocation)); } @@ -308,13 +382,16 @@ void JitCodeCache::ClearGcRootsInInlineCaches(Thread* self) { uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, ArtMethod* method, - const uint8_t* vmap_table, + uint8_t* stack_map, + uint8_t* roots_data, size_t frame_size_in_bytes, size_t core_spill_mask, size_t fp_spill_mask, const uint8_t* code, size_t code_size, - bool osr) { + bool osr, + Handle<mirror::ObjectArray<mirror::Object>> roots) { + DCHECK(stack_map != nullptr); size_t alignment = GetInstructionSetAlignment(kRuntimeISA); // Ensure the header ends up at expected instruction alignment. size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); @@ -338,7 +415,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, std::copy(code, code + code_size, code_ptr); method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); new (method_header) OatQuickMethodHeader( - (vmap_table == nullptr) ? 0 : code_ptr - vmap_table, + code_ptr - stack_map, frame_size_in_bytes, core_spill_mask, fp_spill_mask, @@ -353,6 +430,8 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, { MutexLock mu(self, lock_); method_code_map_.Put(code_ptr, method); + // Fill the root table before updating the entry point. + FillRootTable(roots_data, roots); if (osr) { number_of_osr_compilations_++; osr_code_map_.Put(method, code_ptr); @@ -408,8 +487,14 @@ void JitCodeCache::ClearData(Thread* self, void* data) { FreeData(reinterpret_cast<uint8_t*>(data)); } -uint8_t* JitCodeCache::ReserveData(Thread* self, size_t size, ArtMethod* method) { - size = RoundUp(size, sizeof(void*)); +void JitCodeCache::ReserveData(Thread* self, + size_t stack_map_size, + size_t number_of_roots, + ArtMethod* method, + uint8_t** stack_map_data, + uint8_t** roots_data) { + size_t table_size = ComputeRootTableSize(number_of_roots); + size_t size = RoundUp(stack_map_size + table_size, sizeof(void*)); uint8_t* result = nullptr; { @@ -436,7 +521,8 @@ uint8_t* JitCodeCache::ReserveData(Thread* self, size_t size, ArtMethod* method) << " for stack maps of " << ArtMethod::PrettyMethod(method); } - return result; + *roots_data = result; + *stack_map_data = result + table_size; } class MarkCodeVisitor FINAL : public StackVisitor { diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h index e15c93a448..a97ef683f7 100644 --- a/runtime/jit/jit_code_cache.h +++ b/runtime/jit/jit_code_cache.h @@ -92,13 +92,15 @@ class JitCodeCache { // Allocate and write code and its metadata to the code cache. uint8_t* CommitCode(Thread* self, ArtMethod* method, - const uint8_t* vmap_table, + uint8_t* stack_map, + uint8_t* roots_data, size_t frame_size_in_bytes, size_t core_spill_mask, size_t fp_spill_mask, const uint8_t* code, size_t code_size, - bool osr) + bool osr, + Handle<mirror::ObjectArray<mirror::Object>> roots) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!lock_); @@ -108,8 +110,14 @@ class JitCodeCache { // Return true if the code cache contains this method. bool ContainsMethod(ArtMethod* method) REQUIRES(!lock_); - // Reserve a region of data of size at least "size". Returns null if there is no more room. - uint8_t* ReserveData(Thread* self, size_t size, ArtMethod* method) + // Allocate a region of data that contain `size` bytes, and potentially space + // for storing `number_of_roots` roots. Returns null if there is no more room. + void ReserveData(Thread* self, + size_t size, + size_t number_of_roots, + ArtMethod* method, + uint8_t** stack_map_data, + uint8_t** roots_data) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!lock_); @@ -188,6 +196,10 @@ class JitCodeCache { bool IsOsrCompiled(ArtMethod* method) REQUIRES(!lock_); + void SweepRootTables(IsMarkedVisitor* visitor) + REQUIRES(!lock_) + REQUIRES_SHARED(Locks::mutator_lock_); + private: // Take ownership of maps. JitCodeCache(MemMap* code_map, @@ -201,13 +213,15 @@ class JitCodeCache { // allocation fails. Return null if the allocation fails. uint8_t* CommitCodeInternal(Thread* self, ArtMethod* method, - const uint8_t* vmap_table, + uint8_t* stack_map, + uint8_t* roots_data, size_t frame_size_in_bytes, size_t core_spill_mask, size_t fp_spill_mask, const uint8_t* code, size_t code_size, - bool osr) + bool osr, + Handle<mirror::ObjectArray<mirror::Object>> roots) REQUIRES(!lock_) REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/runtime/oat_quick_method_header.h b/runtime/oat_quick_method_header.h index ee5002f84a..4afca7d828 100644 --- a/runtime/oat_quick_method_header.h +++ b/runtime/oat_quick_method_header.h @@ -67,6 +67,11 @@ class PACKED(4) OatQuickMethodHeader { return data; } + uint8_t* GetOptimizedCodeInfoPtr() { + DCHECK(IsOptimized()); + return code_ - vmap_table_offset_; + } + CodeInfo GetOptimizedCodeInfo() const { return CodeInfo(GetOptimizedCodeInfoPtr()); } diff --git a/runtime/runtime.cc b/runtime/runtime.cc index 1490f05a75..8a3bac77be 100644 --- a/runtime/runtime.cc +++ b/runtime/runtime.cc @@ -81,6 +81,7 @@ #include "intern_table.h" #include "interpreter/interpreter.h" #include "jit/jit.h" +#include "jit/jit_code_cache.h" #include "jni_internal.h" #include "linear_alloc.h" #include "mirror/array.h" @@ -516,6 +517,14 @@ void Runtime::SweepSystemWeaks(IsMarkedVisitor* visitor) { GetMonitorList()->SweepMonitorList(visitor); GetJavaVM()->SweepJniWeakGlobals(visitor); GetHeap()->SweepAllocationRecords(visitor); + if (GetJit() != nullptr) { + // Visit JIT literal tables. Objects in these tables are classes and strings + // and only classes can be affected by class unloading. The strings always + // stay alive as they are strongly interned. + // TODO: Move this closer to CleanupClassLoaders, to avoid blocking weak accesses + // from mutators. See b/32167580. + GetJit()->GetCodeCache()->SweepRootTables(visitor); + } // All other generic system-weak holders. for (gc::AbstractSystemWeakHolder* holder : system_weak_holders_) { diff --git a/test/626-set-resolved-string/expected.txt b/test/626-set-resolved-string/expected.txt new file mode 100644 index 0000000000..f4983b5870 --- /dev/null +++ b/test/626-set-resolved-string/expected.txt @@ -0,0 +1,2 @@ +JNI_OnLoad called +foo diff --git a/test/626-set-resolved-string/info.txt b/test/626-set-resolved-string/info.txt new file mode 100644 index 0000000000..e3a512fd23 --- /dev/null +++ b/test/626-set-resolved-string/info.txt @@ -0,0 +1,3 @@ +Test that even if Java code calls DexCache.setResolvedString and does +not strongly intern the given string, the JIT will ensure that the +strings it references are strongly interned. diff --git a/test/626-set-resolved-string/src/Main.java b/test/626-set-resolved-string/src/Main.java new file mode 100644 index 0000000000..868b9d1e40 --- /dev/null +++ b/test/626-set-resolved-string/src/Main.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.lang.reflect.Method; + +public class Main { + public static void main(String[] args) { + System.loadLibrary(args[0]); + + // Get all methods. We cannot call getDeclaredMethod("foo") as + // that would make "foo" a strong root. + Method[] methods = Main.class.getDeclaredMethods(); + + // Call getName on the methods, which is implemented by using the dex + // cache and calling setResolvedString. + for (int i = 0; i < methods.length; i++) { + methods[i].getName(); + } + + // Compile Main.foo. "foo" needs to be a strong root for JIT compilation. + // We stress test this: + // - avoid strongly interning "foo" by doing "f" + "oo" + // - call GC so that weaks can be collected. + // - invoke foo() to make sure "foo" hasn't been collected. + ensureJitCompiled(Main.class, "f" + "oo"); + Runtime.getRuntime().gc(); + foo(); + } + + public static void foo() { + System.out.println("foo"); + } + + public static native void ensureJitCompiled(Class cls, String method_name); +} |