diff options
42 files changed, 1028 insertions, 315 deletions
diff --git a/build/Android.gtest.mk b/build/Android.gtest.mk index c4374f7101..53891b13a9 100644 --- a/build/Android.gtest.mk +++ b/build/Android.gtest.mk @@ -171,6 +171,12 @@ ART_GTEST_dex2oat_test_TARGET_DEPS := \ # TODO: document why this is needed. ART_GTEST_proxy_test_HOST_DEPS := $(HOST_CORE_IMAGE_DEFAULT_64) $(HOST_CORE_IMAGE_DEFAULT_32) +# The dexdiag test requires the dexdiag utility. +ART_GTEST_dexdiag_test_HOST_DEPS := \ + $(HOST_OUT_EXECUTABLES)/dexdiag +ART_GTEST_dexdiag_test_TARGET_DEPS := \ + dexdiag + # The dexdump test requires an image and the dexdump utility. # TODO: rename into dexdump when migration completes ART_GTEST_dexdump_test_HOST_DEPS := \ @@ -244,6 +250,7 @@ ART_TEST_MODULES := \ art_compiler_tests \ art_compiler_host_tests \ art_dex2oat_tests \ + art_dexdiag_tests \ art_dexdump_tests \ art_dexlayout_tests \ art_dexlist_tests \ diff --git a/compiler/optimizing/bounds_check_elimination.cc b/compiler/optimizing/bounds_check_elimination.cc index 476906a768..d38d5f8684 100644 --- a/compiler/optimizing/bounds_check_elimination.cc +++ b/compiler/optimizing/bounds_check_elimination.cc @@ -1749,6 +1749,7 @@ class BCEVisitor : public HGraphVisitor { phi = NewPhi(new_preheader, instruction, type); } user->ReplaceInput(phi, index); // Removes the use node from the list. + induction_range_.Replace(user, instruction, phi); // update induction } } // Scan all environment uses of an instruction and replace each later use with a phi node. diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 8744cc8210..d65b3276c8 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -8521,10 +8521,6 @@ VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateBootImageAddressLiteral(uint return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); } -VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateDexCacheAddressLiteral(uint32_t address) { - return DeduplicateUint32Literal(address, &uint32_literals_); -} - VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateJitStringLiteral( const DexFile& dex_file, dex::StringIndex string_index, diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index 1e9669dc38..7281069102 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -577,7 +577,6 @@ class CodeGeneratorARMVIXL : public CodeGenerator { VIXLUInt32Literal* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, dex::TypeIndex type_index); VIXLUInt32Literal* DeduplicateBootImageAddressLiteral(uint32_t address); - VIXLUInt32Literal* DeduplicateDexCacheAddressLiteral(uint32_t address); VIXLUInt32Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index, Handle<mirror::String> handle); diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 9736626b70..357df9799c 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -219,15 +219,33 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); - + const bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); + const bool r2_baker_or_no_read_barriers = !isR6 && (!kUseReadBarrier || kUseBakerReadBarrier); + InvokeRuntimeCallingConvention calling_convention; + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + const bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + Register entry_address = kNoRegister; + if (is_load_class_bss_entry && r2_baker_or_no_read_barriers) { + Register temp = locations->GetTemp(0).AsRegister<Register>(); + bool temp_is_a0 = (temp == calling_convention.GetRegisterAt(0)); + // In the unlucky case that `temp` is A0, we preserve the address in `out` across the + // kSaveEverything call. + entry_address = temp_is_a0 ? out.AsRegister<Register>() : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_a0) { + __ Move(entry_address, temp); + } + } + dex::TypeIndex type_index = cls_->GetTypeIndex(); __ LoadConst32(calling_convention.GetRegisterAt(0), type_index.index_); - QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage : kQuickInitializeType; mips_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); @@ -237,25 +255,27 @@ class LoadClassSlowPathMIPS : public SlowPathCodeMIPS { CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); } + // For HLoadClass/kBssEntry, store the resolved class to the BSS entry. + if (is_load_class_bss_entry && r2_baker_or_no_read_barriers) { + // The class entry address was preserved in `entry_address` thanks to kSaveEverything. + __ StoreToOffset(kStoreWord, calling_convention.GetRegisterAt(0), entry_address, 0); + } + // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); Primitive::Type type = instruction_->GetType(); - mips_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type); + mips_codegen->MoveLocation(out, + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + type); } - RestoreLiveRegisters(codegen, locations); - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { - DCHECK(out.IsValid()); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the .bss entry address in the fast path, - // so that we can avoid another calculation here. - bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); + + // For HLoadClass/kBssEntry, store the resolved class to the BSS entry. + if (is_load_class_bss_entry && !r2_baker_or_no_read_barriers) { + // For non-Baker read barriers (or on R6), we need to re-calculate the address of + // the class entry. Register base = isR6 ? ZERO : locations->InAt(0).AsRegister<Register>(); - DCHECK_NE(out.AsRegister<Register>(), AT); CodeGeneratorMIPS::PcRelativePatchInfo* info = mips_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); bool reordering = __ SetReorder(false); @@ -286,40 +306,62 @@ class LoadStringSlowPathMIPS : public SlowPathCodeMIPS { explicit LoadStringSlowPathMIPS(HLoadString* instruction) : SlowPathCodeMIPS(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + DCHECK(instruction_->IsLoadString()); + DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); LocationSummary* locations = instruction_->GetLocations(); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); + HLoadString* load = instruction_->AsLoadString(); + const dex::StringIndex string_index = load->GetStringIndex(); + Register out = locations->Out().AsRegister<Register>(); CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); - + const bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); + const bool r2_baker_or_no_read_barriers = !isR6 && (!kUseReadBarrier || kUseBakerReadBarrier); + InvokeRuntimeCallingConvention calling_convention; __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; - HLoadString* load = instruction_->AsLoadString(); - const dex::StringIndex string_index = load->GetStringIndex(); + // For HLoadString/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + Register entry_address = kNoRegister; + if (r2_baker_or_no_read_barriers) { + Register temp = locations->GetTemp(0).AsRegister<Register>(); + bool temp_is_a0 = (temp == calling_convention.GetRegisterAt(0)); + // In the unlucky case that `temp` is A0, we preserve the address in `out` across the + // kSaveEverything call. + entry_address = temp_is_a0 ? out : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_a0) { + __ Move(entry_address, temp); + } + } + __ LoadConst32(calling_convention.GetRegisterAt(0), string_index.index_); mips_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); + + // Store the resolved string to the BSS entry. + if (r2_baker_or_no_read_barriers) { + // The string entry address was preserved in `entry_address` thanks to kSaveEverything. + __ StoreToOffset(kStoreWord, calling_convention.GetRegisterAt(0), entry_address, 0); + } + Primitive::Type type = instruction_->GetType(); mips_codegen->MoveLocation(locations->Out(), - calling_convention.GetReturnLocation(type), + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), type); - RestoreLiveRegisters(codegen, locations); - // Store the resolved String to the BSS entry. - // TODO: Change art_quick_resolve_string to kSaveEverything and use a temporary for the - // .bss entry address in the fast path, so that we can avoid another calculation here. - bool isR6 = mips_codegen->GetInstructionSetFeatures().IsR6(); - Register base = isR6 ? ZERO : locations->InAt(0).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - DCHECK_NE(out, AT); - CodeGeneratorMIPS::PcRelativePatchInfo* info = - mips_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); - bool reordering = __ SetReorder(false); - mips_codegen->EmitPcRelativeAddressPlaceholderHigh(info, TMP, base); - __ StoreToOffset(kStoreWord, out, TMP, /* placeholder */ 0x5678); - __ SetReorder(reordering); - + // Store the resolved string to the BSS entry. + if (!r2_baker_or_no_read_barriers) { + // For non-Baker read barriers (or on R6), we need to re-calculate the address of + // the string entry. + Register base = isR6 ? ZERO : locations->InAt(0).AsRegister<Register>(); + CodeGeneratorMIPS::PcRelativePatchInfo* info = + mips_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); + bool reordering = __ SetReorder(false); + mips_codegen->EmitPcRelativeAddressPlaceholderHigh(info, TMP, base); + __ StoreToOffset(kStoreWord, out, TMP, /* placeholder */ 0x5678); + __ SetReorder(reordering); + } __ B(GetExitLabel()); } @@ -1719,15 +1761,14 @@ void CodeGeneratorMIPS::PatchJitRootUse(uint8_t* code, DCHECK_EQ(code[literal_offset + 1], 0x12); DCHECK_EQ((code[literal_offset + 2] & 0xE0), 0x00); DCHECK_EQ(code[literal_offset + 3], 0x3C); - // lw reg, reg, addr32_low + // instr reg, reg, addr32_low DCHECK_EQ(code[literal_offset + 4], 0x78); DCHECK_EQ(code[literal_offset + 5], 0x56); - DCHECK_EQ((code[literal_offset + 7] & 0xFC), 0x8C); - addr32 += (addr32 & 0x8000) << 1; // Account for sign extension in "lw reg, reg, addr32_low". + addr32 += (addr32 & 0x8000) << 1; // Account for sign extension in "instr reg, reg, addr32_low". // lui reg, addr32_high code[literal_offset + 0] = static_cast<uint8_t>(addr32 >> 16); code[literal_offset + 1] = static_cast<uint8_t>(addr32 >> 24); - // lw reg, reg, addr32_low + // instr reg, reg, addr32_low code[literal_offset + 4] = static_cast<uint8_t>(addr32 >> 0); code[literal_offset + 5] = static_cast<uint8_t>(addr32 >> 8); } @@ -2436,6 +2477,9 @@ void LocationsBuilderMIPS::VisitArrayGet(HArrayGet* instruction) { object_array_get_with_read_barrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall); + if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); if (Primitive::IsFloatingPointType(type)) { @@ -5767,6 +5811,9 @@ void LocationsBuilderMIPS::HandleFieldGet(HInstruction* instruction, const Field ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall)); + if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::RequiresRegister()); if (generate_volatile) { InvokeRuntimeCallingConvention calling_convention; @@ -6445,6 +6492,7 @@ void CodeGeneratorMIPS::GenerateReadBarrierForRootSlow(HInstruction* instruction void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); + bool baker_read_barrier_slow_path = false; switch (type_check_kind) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kAbstractClassCheck: @@ -6452,6 +6500,7 @@ void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kArrayObjectCheck: call_kind = kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; + baker_read_barrier_slow_path = kUseBakerReadBarrier; break; case TypeCheckKind::kArrayCheck: case TypeCheckKind::kUnresolvedCheck: @@ -6461,6 +6510,9 @@ void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { } LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); + if (baker_read_barrier_slow_path) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); // The output does overlap inputs. @@ -7048,26 +7100,27 @@ void LocationsBuilderMIPS::VisitLoadClass(HLoadClass* cls) { HLoadClass::LoadKind load_kind = cls->GetLoadKind(); if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { InvokeRuntimeCallingConvention calling_convention; - CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( - cls, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - calling_convention.GetReturnLocation(Primitive::kPrimNot)); + Location loc = Location::RegisterLocation(calling_convention.GetRegisterAt(0)); + CodeGenerator::CreateLoadClassRuntimeCallLocationSummary(cls, loc, loc); return; } DCHECK(!cls->NeedsAccessCheck()); - + const bool isR6 = codegen_->GetInstructionSetFeatures().IsR6(); const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); + if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } switch (load_kind) { // We need an extra register for PC-relative literals on R2. case HLoadClass::LoadKind::kBootImageLinkTimeAddress: case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: case HLoadClass::LoadKind::kBootImageAddress: case HLoadClass::LoadKind::kBssEntry: - if (codegen_->GetInstructionSetFeatures().IsR6()) { + if (isR6) { break; } FALLTHROUGH_INTENDED; @@ -7078,6 +7131,22 @@ void LocationsBuilderMIPS::VisitLoadClass(HLoadClass* cls) { break; } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + // Request a temp to hold the BSS entry location for the slow path on R2 + // (no benefit for R6). + if (!isR6) { + locations->AddTemp(Location::RequiresRegister()); + } + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barriers we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -7160,10 +7229,22 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF case HLoadClass::LoadKind::kBssEntry: { CodeGeneratorMIPS::PcRelativePatchInfo* info = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); - bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); - GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678, read_barrier_option); - __ SetReorder(reordering); + constexpr bool non_baker_read_barrier = kUseReadBarrier && !kUseBakerReadBarrier; + if (isR6 || non_baker_read_barrier) { + bool reordering = __ SetReorder(false); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); + GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678, read_barrier_option); + __ SetReorder(reordering); + } else { + // On R2 save the BSS entry address in a temporary register instead of + // recalculating it in the slow path. + Register temp = locations->GetTemp(0).AsRegister<Register>(); + bool reordering = __ SetReorder(false); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, temp, base_or_current_method_reg); + __ Addiu(temp, temp, /* placeholder */ 0x5678); + __ SetReorder(reordering); + GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); + } generate_null_check = true; break; } @@ -7227,13 +7308,14 @@ void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); HLoadString::LoadKind load_kind = load->GetLoadKind(); + const bool isR6 = codegen_->GetInstructionSetFeatures().IsR6(); switch (load_kind) { // We need an extra register for PC-relative literals on R2. case HLoadString::LoadKind::kBootImageLinkTimeAddress: case HLoadString::LoadKind::kBootImageAddress: case HLoadString::LoadKind::kBootImageLinkTimePcRelative: case HLoadString::LoadKind::kBssEntry: - if (codegen_->GetInstructionSetFeatures().IsR6()) { + if (isR6) { break; } FALLTHROUGH_INTENDED; @@ -7246,9 +7328,25 @@ void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { } if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { InvokeRuntimeCallingConvention calling_convention; - locations->SetOut(calling_convention.GetReturnLocation(load->GetType())); + locations->SetOut(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); } else { locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadString::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the pResolveString and marking to save everything we need. + // Request a temp to hold the BSS entry location for the slow path on R2 + // (no benefit for R6). + if (!isR6) { + locations->AddTemp(Location::RequiresRegister()); + } + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barriers we have a temp-clobbering call. + } + } } } @@ -7305,14 +7403,26 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); CodeGeneratorMIPS::PcRelativePatchInfo* info = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); - bool reordering = __ SetReorder(false); - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); - GenerateGcRootFieldLoad(load, - out_loc, - out, - /* placeholder */ 0x5678, - kCompilerReadBarrierOption); - __ SetReorder(reordering); + constexpr bool non_baker_read_barrier = kUseReadBarrier && !kUseBakerReadBarrier; + if (isR6 || non_baker_read_barrier) { + bool reordering = __ SetReorder(false); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); + GenerateGcRootFieldLoad(load, + out_loc, + out, + /* placeholder */ 0x5678, + kCompilerReadBarrierOption); + __ SetReorder(reordering); + } else { + // On R2 save the BSS entry address in a temporary register instead of + // recalculating it in the slow path. + Register temp = locations->GetTemp(0).AsRegister<Register>(); + bool reordering = __ SetReorder(false); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, temp, base_or_current_method_reg); + __ Addiu(temp, temp, /* placeholder */ 0x5678); + __ SetReorder(reordering); + GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); + } SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathMIPS(load); codegen_->AddSlowPath(slow_path); __ Beqz(out, slow_path->GetEntryLabel()); @@ -7342,6 +7452,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ // TODO: Re-add the compiler code to do string dex cache lookup again. DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); InvokeRuntimeCallingConvention calling_convention; + DCHECK_EQ(calling_convention.GetRegisterAt(0), out); __ LoadConst32(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 6f37ed44c4..a9c4964a89 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -141,7 +141,8 @@ class BoundsCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { class DivZeroCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { public: - explicit DivZeroCheckSlowPathMIPS64(HDivZeroCheck* instruction) : SlowPathCodeMIPS64(instruction) {} + explicit DivZeroCheckSlowPathMIPS64(HDivZeroCheck* instruction) + : SlowPathCodeMIPS64(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); @@ -192,7 +193,9 @@ class LoadClassSlowPathMIPS64 : public SlowPathCodeMIPS64 { if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); Primitive::Type type = instruction_->GetType(); - mips64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type); + mips64_codegen->MoveLocation(out, + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + type); } RestoreLiveRegisters(codegen, locations); @@ -200,10 +203,6 @@ class LoadClassSlowPathMIPS64 : public SlowPathCodeMIPS64 { DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { DCHECK(out.IsValid()); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the .bss entry address in the fast path, - // so that we can avoid another calculation here. - DCHECK_NE(out.AsRegister<GpuRegister>(), AT); CodeGeneratorMIPS64::PcRelativePatchInfo* info = mips64_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); mips64_codegen->EmitPcRelativeAddressPlaceholderHigh(info, AT); @@ -250,16 +249,13 @@ class LoadStringSlowPathMIPS64 : public SlowPathCodeMIPS64 { CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); Primitive::Type type = instruction_->GetType(); mips64_codegen->MoveLocation(locations->Out(), - calling_convention.GetReturnLocation(type), + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), type); RestoreLiveRegisters(codegen, locations); // Store the resolved String to the BSS entry. - // TODO: Change art_quick_resolve_string to kSaveEverything and use a temporary for the - // .bss entry address in the fast path, so that we can avoid another calculation here. GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - DCHECK_NE(out, AT); CodeGeneratorMIPS64::PcRelativePatchInfo* info = mips64_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); mips64_codegen->EmitPcRelativeAddressPlaceholderHigh(info, AT); @@ -1986,6 +1982,9 @@ void LocationsBuilderMIPS64::VisitArrayGet(HArrayGet* instruction) { object_array_get_with_read_barrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall); + if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); if (Primitive::IsFloatingPointType(type)) { @@ -3982,6 +3981,9 @@ void LocationsBuilderMIPS64::HandleFieldGet(HInstruction* instruction, object_field_get_with_read_barrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall); + if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::RequiresRegister()); if (Primitive::IsFloatingPointType(instruction->GetType())) { locations->SetOut(Location::RequiresFpuRegister()); @@ -4544,6 +4546,7 @@ void CodeGeneratorMIPS64::GenerateReadBarrierForRootSlow(HInstruction* instructi void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); + bool baker_read_barrier_slow_path = false; switch (type_check_kind) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kAbstractClassCheck: @@ -4551,6 +4554,7 @@ void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kArrayObjectCheck: call_kind = kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; + baker_read_barrier_slow_path = kUseBakerReadBarrier; break; case TypeCheckKind::kArrayCheck: case TypeCheckKind::kUnresolvedCheck: @@ -4560,6 +4564,9 @@ void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { } LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); + if (baker_read_barrier_slow_path) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); // The output does overlap inputs. @@ -5077,10 +5084,8 @@ void LocationsBuilderMIPS64::VisitLoadClass(HLoadClass* cls) { HLoadClass::LoadKind load_kind = cls->GetLoadKind(); if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { InvokeRuntimeCallingConvention calling_convention; - CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( - cls, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - calling_convention.GetReturnLocation(Primitive::kPrimNot)); + Location loc = Location::RegisterLocation(calling_convention.GetRegisterAt(0)); + CodeGenerator::CreateLoadClassRuntimeCallLocationSummary(cls, loc, loc); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -5090,10 +5095,24 @@ void LocationsBuilderMIPS64::VisitLoadClass(HLoadClass* cls) { ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); + if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } if (load_kind == HLoadClass::LoadKind::kReferrersClass) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -5224,9 +5243,20 @@ void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { InvokeRuntimeCallingConvention calling_convention; - locations->SetOut(calling_convention.GetReturnLocation(load->GetType())); + locations->SetOut(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); } else { locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadString::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the pResolveString and marking to save everything we need. + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } } @@ -5294,6 +5324,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA // TODO: Re-add the compiler code to do string dex cache lookup again. DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); InvokeRuntimeCallingConvention calling_convention; + DCHECK_EQ(calling_convention.GetRegisterAt(0), out); __ LoadConst32(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index e8d0f453a1..3b681c13a5 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -817,7 +817,17 @@ HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, } const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); - bool is_referrer = (klass.Get() == outermost_graph_->GetArtMethod()->GetDeclaringClass()); + bool is_referrer; + ArtMethod* outermost_art_method = outermost_graph_->GetArtMethod(); + if (outermost_art_method == nullptr) { + DCHECK(Runtime::Current()->IsAotCompiler()); + // We are in AOT mode and we don't have an ART method to determine + // if the inlined method belongs to the referrer. Assume it doesn't. + is_referrer = false; + } else { + is_referrer = klass.Get() == outermost_art_method->GetDeclaringClass(); + } + // Note that we will just compare the classes, so we don't need Java semantics access checks. // Note that the type index and the dex file are relative to the method this type guard is // inlined into. diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index bfe04f5ae0..abf5b122c8 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -1525,6 +1525,9 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall), kIntrinsified); + if (can_call && kUseBakerReadBarrier) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index c5e116046f..9dce59b2af 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -1168,6 +1168,9 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall), kIntrinsified); + if (can_call && kUseBakerReadBarrier) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index 9fd42d2cb7..58f394896a 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -2433,8 +2433,8 @@ class Dex2Oat FINAL { // which uses an unstarted runtime. raw_options.push_back(std::make_pair("-Xgc:nonconcurrent", nullptr)); - // Also force the free-list implementation for large objects. - raw_options.push_back(std::make_pair("-XX:LargeObjectSpace=freelist", nullptr)); + // The default LOS implementation (map) is not deterministic. So disable it. + raw_options.push_back(std::make_pair("-XX:LargeObjectSpace=disabled", nullptr)); // We also need to turn off the nonmoving space. For that, we need to disable HSpace // compaction (done above) and ensure that neither foreground nor background collectors diff --git a/dexlayout/Android.bp b/dexlayout/Android.bp index a2116cdc93..588a3ae3ca 100644 --- a/dexlayout/Android.bp +++ b/dexlayout/Android.bp @@ -20,7 +20,7 @@ art_cc_defaults { "dexlayout.cc", "dex_ir.cc", "dex_ir_builder.cc", - "dex_verify.cc", + "dex_verify.cc", "dex_visualize.cc", "dex_writer.cc", ], @@ -43,6 +43,7 @@ art_cc_library { art_cc_binary { name: "dexlayout", + defaults: ["art_defaults"], host_supported: true, srcs: ["dexlayout_main.cc"], cflags: ["-Wall"], @@ -61,13 +62,28 @@ art_cc_test { art_cc_binary { name: "dexdiag", - host_supported: false, + defaults: ["art_defaults"], + host_supported: true, srcs: ["dexdiag.cc"], cflags: ["-Wall"], shared_libs: [ "libart", "libart-dexlayout", - "libpagemap", ], + target: { + android: { + shared_libs: [ + "libpagemap", + ] + }, + } } +art_cc_test { + name: "art_dexdiag_tests", + host_supported: true, + defaults: [ + "art_gtest_defaults", + ], + srcs: ["dexdiag_test.cc"], +} diff --git a/dexlayout/dexdiag.cc b/dexlayout/dexdiag.cc index ea2679a1e3..49c818593b 100644 --- a/dexlayout/dexdiag.cc +++ b/dexlayout/dexdiag.cc @@ -15,6 +15,7 @@ */ #include <errno.h> +#include <inttypes.h> #include <stdint.h> #include <stdlib.h> #include <string.h> @@ -30,7 +31,9 @@ #include "dex_file.h" #include "dex_ir.h" #include "dex_ir_builder.h" +#ifdef ART_TARGET_ANDROID #include "pagemap/pagemap.h" +#endif #include "runtime.h" #include "vdex_file.h" @@ -38,8 +41,6 @@ namespace art { using android::base::StringPrintf; -static constexpr size_t kLineLength = 32; - static bool g_verbose = false; // The width needed to print a file page offset (32-bit). @@ -164,6 +165,7 @@ static void PrintLetterKey() { std::cout << ". (Mapped page not resident)" << std::endl; } +#ifdef ART_TARGET_ANDROID static char PageTypeChar(uint16_t type) { if (kDexSectionInfoMap.find(type) == kDexSectionInfoMap.end()) { return '-'; @@ -194,6 +196,7 @@ static void ProcessPageMap(uint64_t* pagemap, size_t end, const std::vector<dex_ir::DexFileSection>& sections, PageCount* page_counts) { + static constexpr size_t kLineLength = 32; for (size_t page = start; page < end; ++page) { char type_char = '.'; if (PM_PAGEMAP_PRESENT(pagemap[page])) { @@ -268,7 +271,7 @@ static void ProcessOneDexMapping(uint64_t* pagemap, std::cerr << "Dex file start offset for " << dex_file->GetLocation().c_str() << " is incorrect: map start " - << StringPrintf("%zx > dex start %zx\n", map_start, dex_file_start) + << StringPrintf("%" PRIx64 " > dex start %" PRIx64 "\n", map_start, dex_file_start) << std::endl; return; } @@ -277,7 +280,7 @@ static void ProcessOneDexMapping(uint64_t* pagemap, uint64_t end_page = RoundUp(start_address + dex_file_size, kPageSize) / kPageSize; std::cout << "DEX " << dex_file->GetLocation().c_str() - << StringPrintf(": %zx-%zx", + << StringPrintf(": %" PRIx64 "-%" PRIx64, map_start + start_page * kPageSize, map_start + end_page * kPageSize) << std::endl; @@ -342,7 +345,7 @@ static bool DisplayMappingIfFromVdexFile(pm_map_t* map, Printer* printer) { // Process the dex files. std::cout << "MAPPING " << pm_map_name(map) - << StringPrintf(": %zx-%zx", pm_map_start(map), pm_map_end(map)) + << StringPrintf(": %" PRIx64 "-%" PRIx64, pm_map_start(map), pm_map_end(map)) << std::endl; for (const auto& dex_file : dex_files) { ProcessOneDexMapping(pagemap, @@ -356,6 +359,7 @@ static bool DisplayMappingIfFromVdexFile(pm_map_t* map, Printer* printer) { } static void ProcessOneOatMapping(uint64_t* pagemap, size_t size, Printer* printer) { + static constexpr size_t kLineLength = 32; size_t resident_page_count = 0; for (size_t page = 0; page < size; ++page) { char type_char = '.'; @@ -406,7 +410,7 @@ static bool DisplayMappingIfFromOatFile(pm_map_t* map, Printer* printer) { // Process the dex files. std::cout << "MAPPING " << pm_map_name(map) - << StringPrintf(": %zx-%zx", pm_map_start(map), pm_map_end(map)) + << StringPrintf(": %" PRIx64 "-%" PRIx64, pm_map_start(map), pm_map_end(map)) << std::endl; ProcessOneOatMapping(pagemap, len, printer); free(pagemap); @@ -426,9 +430,10 @@ static bool FilterByNameContains(const std::string& mapped_file_name, } return false; } +#endif static void Usage(const char* cmd) { - std::cerr << "Usage: " << cmd << " [options] pid" << std::endl + std::cout << "Usage: " << cmd << " [options] pid" << std::endl << " --contains=<string>: Display sections containing string." << std::endl << " --help: Shows this message." << std::endl << " --verbose: Makes displays verbose." << std::endl; @@ -463,6 +468,7 @@ static int DexDiagMain(int argc, char* argv[]) { InitLogging(argv, Runtime::Aborter); MemMap::Init(); +#ifdef ART_TARGET_ANDROID pid_t pid; char* endptr; pid = (pid_t)strtol(argv[argc - 1], &endptr, 10); @@ -496,7 +502,8 @@ static int DexDiagMain(int argc, char* argv[]) { return EXIT_FAILURE; } - // Process the mappings that are due to DEX files. + bool match_found = false; + // Process the mappings that are due to vdex or oat files. Printer printer; for (size_t i = 0; i < num_maps; ++i) { std::string mapped_file_name = pm_map_name(maps[i]); @@ -504,12 +511,17 @@ static int DexDiagMain(int argc, char* argv[]) { if (!FilterByNameContains(mapped_file_name, name_filters)) { continue; } + match_found = true; if (!DisplayMappingIfFromVdexFile(maps[i], &printer)) { return EXIT_FAILURE; } else if (!DisplayMappingIfFromOatFile(maps[i], &printer)) { return EXIT_FAILURE; } } + if (!match_found) { + return EXIT_FAILURE; + } +#endif return EXIT_SUCCESS; } diff --git a/dexlayout/dexdiag_test.cc b/dexlayout/dexdiag_test.cc new file mode 100644 index 0000000000..d0d2af14e5 --- /dev/null +++ b/dexlayout/dexdiag_test.cc @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <vector> + +#include "common_runtime_test.h" + +#include "runtime/exec_utils.h" +#include "runtime/oat_file.h" +#include "runtime/os.h" + +namespace art { + +static const char* kDexDiagContains = "--contains=core.vdex"; +static const char* kDexDiagContainsFails = "--contains=anything_other_than_core.vdex"; +static const char* kDexDiagHelp = "--help"; +static const char* kDexDiagVerbose = "--verbose"; +static const char* kDexDiagBinaryName = "dexdiag"; + +class DexDiagTest : public CommonRuntimeTest { + protected: + virtual void SetUp() { + CommonRuntimeTest::SetUp(); + } + + // Path to the dexdiag(d?)[32|64] binary. + std::string GetDexDiagFilePath() { + std::string root = GetTestAndroidRoot(); + + root += "/bin/"; + root += kDexDiagBinaryName; + + std::string root32 = root + "32"; + // If we have both a 32-bit and a 64-bit build, the 32-bit file will have a 32 suffix. + if (OS::FileExists(root32.c_str()) && !Is64BitInstructionSet(kRuntimeISA)) { + return root32; + } else { + // This is a 64-bit build or only a single build exists. + return root; + } + } + + std::unique_ptr<OatFile> OpenOatAndVdexFiles() { + std::cout << "YO!" << std::endl; + // Open the core.oat file. + // This is a little convoluted because we have to + // get the location of the default core image (.../framework/core.oat), + // find it in the right architecture subdirectory (.../framework/arm/core.oat), + // Then, opening the oat file has the side-effect of opening the corresponding + // vdex file (.../framework/arm/core.vdex). + const std::string default_location = GetCoreOatLocation(); + EXPECT_TRUE(!default_location.empty()); + std::string oat_location = GetSystemImageFilename(default_location.c_str(), kRuntimeISA); + EXPECT_TRUE(!oat_location.empty()); + std::cout << "==" << oat_location << std::endl; + std::string error_msg; + std::unique_ptr<OatFile> oat(OatFile::Open(oat_location.c_str(), + oat_location.c_str(), + nullptr, + nullptr, + false, + /*low_4gb*/false, + nullptr, + &error_msg)); + EXPECT_TRUE(oat != nullptr) << error_msg; + return oat; + } + + // Run dexdiag with a custom boot image location. + bool Exec(pid_t this_pid, const std::vector<std::string>& args, std::string* error_msg) { + // Invoke 'dexdiag' against the current process. + // This should succeed because we have a runtime and so it should + // be able to map in the boot.art and do a diff for it. + std::vector<std::string> exec_argv; + + // Build the command line "dexdiag <args> this_pid". + std::string executable_path = GetDexDiagFilePath(); + EXPECT_TRUE(OS::FileExists(executable_path.c_str())) << executable_path + << " should be a valid file path"; + exec_argv.push_back(executable_path); + for (const auto& arg : args) { + exec_argv.push_back(arg); + } + exec_argv.push_back(std::to_string(this_pid)); + + return ::art::Exec(exec_argv, error_msg); + } +}; + +// We can't run these tests on the host, as they will fail when trying to open +// /proc/pid/pagemap. +// On the target, we invoke 'dexdiag' against the current process. +// This should succeed because we have a runtime and so dexdiag should +// be able to find the map for, e.g., boot.vdex and friends. +TEST_F(DexDiagTest, DexDiagHelpTest) { + // TODO: test the resulting output. + std::string error_msg; + ASSERT_TRUE(Exec(getpid(), { kDexDiagHelp }, &error_msg)) << "Failed to execute -- because: " + << error_msg; +} + +#if defined (ART_TARGET) +TEST_F(DexDiagTest, DexDiagContainsTest) { +#else +TEST_F(DexDiagTest, DISABLED_DexDiagContainsTest) { +#endif + std::unique_ptr<OatFile> oat = OpenOatAndVdexFiles(); + // TODO: test the resulting output. + std::string error_msg; + ASSERT_TRUE(Exec(getpid(), { kDexDiagContains }, &error_msg)) << "Failed to execute -- because: " + << error_msg; +} + +#if defined (ART_TARGET) +TEST_F(DexDiagTest, DexDiagContainsFailsTest) { +#else +TEST_F(DexDiagTest, DISABLED_DexDiagContainsFailsTest) { +#endif + std::unique_ptr<OatFile> oat = OpenOatAndVdexFiles(); + // TODO: test the resulting output. + std::string error_msg; + ASSERT_FALSE(Exec(getpid(), { kDexDiagContainsFails }, &error_msg)) + << "Failed to execute -- because: " + << error_msg; +} + +#if defined (ART_TARGET) +TEST_F(DexDiagTest, DexDiagVerboseTest) { +#else +TEST_F(DexDiagTest, DISABLED_DexDiagVerboseTest) { +#endif + // TODO: test the resulting output. + std::unique_ptr<OatFile> oat = OpenOatAndVdexFiles(); + std::string error_msg; + ASSERT_TRUE(Exec(getpid(), { kDexDiagVerbose }, &error_msg)) << "Failed to execute -- because: " + << error_msg; +} + +} // namespace art diff --git a/runtime/arch/arm/entrypoints_init_arm.cc b/runtime/arch/arm/entrypoints_init_arm.cc index de72d3a18f..d21d0c07a2 100644 --- a/runtime/arch/arm/entrypoints_init_arm.cc +++ b/runtime/arch/arm/entrypoints_init_arm.cc @@ -67,19 +67,19 @@ extern "C" int __aeabi_idivmod(int32_t, int32_t); // [DIV|REM]_INT[_2ADDR|_LIT8 // Long long arithmetics - REM_LONG[_2ADDR] and DIV_LONG[_2ADDR] extern "C" int64_t __aeabi_ldivmod(int64_t, int64_t); -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { - qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr; - qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; - qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; - qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; - qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr; - qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; - qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; - qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; - qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr; - qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr; - qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr; - qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr; +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_active) { + qpoints->pReadBarrierMarkReg00 = is_active ? art_quick_read_barrier_mark_reg00 : nullptr; + qpoints->pReadBarrierMarkReg01 = is_active ? art_quick_read_barrier_mark_reg01 : nullptr; + qpoints->pReadBarrierMarkReg02 = is_active ? art_quick_read_barrier_mark_reg02 : nullptr; + qpoints->pReadBarrierMarkReg03 = is_active ? art_quick_read_barrier_mark_reg03 : nullptr; + qpoints->pReadBarrierMarkReg04 = is_active ? art_quick_read_barrier_mark_reg04 : nullptr; + qpoints->pReadBarrierMarkReg05 = is_active ? art_quick_read_barrier_mark_reg05 : nullptr; + qpoints->pReadBarrierMarkReg06 = is_active ? art_quick_read_barrier_mark_reg06 : nullptr; + qpoints->pReadBarrierMarkReg07 = is_active ? art_quick_read_barrier_mark_reg07 : nullptr; + qpoints->pReadBarrierMarkReg08 = is_active ? art_quick_read_barrier_mark_reg08 : nullptr; + qpoints->pReadBarrierMarkReg09 = is_active ? art_quick_read_barrier_mark_reg09 : nullptr; + qpoints->pReadBarrierMarkReg10 = is_active ? art_quick_read_barrier_mark_reg10 : nullptr; + qpoints->pReadBarrierMarkReg11 = is_active ? art_quick_read_barrier_mark_reg11 : nullptr; } void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { @@ -138,7 +138,7 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + UpdateReadBarrierEntrypoints(qpoints, /*is_active*/ false); qpoints->pReadBarrierMarkReg12 = nullptr; // Cannot use register 12 (IP) to pass arguments. qpoints->pReadBarrierMarkReg13 = nullptr; // Cannot use register 13 (SP) to pass arguments. qpoints->pReadBarrierMarkReg14 = nullptr; // Cannot use register 14 (LR) to pass arguments. diff --git a/runtime/arch/arm64/entrypoints_init_arm64.cc b/runtime/arch/arm64/entrypoints_init_arm64.cc index bc7bcb1739..610cdee683 100644 --- a/runtime/arch/arm64/entrypoints_init_arm64.cc +++ b/runtime/arch/arm64/entrypoints_init_arm64.cc @@ -75,7 +75,7 @@ extern "C" mirror::Object* art_quick_read_barrier_mark_introspection(mirror::Obj extern "C" mirror::Object* art_quick_read_barrier_mark_introspection_arrays(mirror::Object*); extern "C" mirror::Object* art_quick_read_barrier_mark_introspection_gc_roots(mirror::Object*); -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_active) { // ARM64 is the architecture with the largest number of core // registers (32) that supports the read barrier configuration. // Because registers 30 (LR) and 31 (SP/XZR) cannot be used to pass @@ -85,35 +85,35 @@ void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { // have less core registers (resp. 16, 8 and 16). (We may have to // revise that design choice if read barrier support is added for // MIPS and/or MIPS64.) - qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr; - qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; - qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; - qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; - qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr; - qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; - qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; - qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; - qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr; - qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr; - qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr; - qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr; - qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr; - qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr; - qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr; - qpoints->pReadBarrierMarkReg15 = is_marking ? art_quick_read_barrier_mark_reg15 : nullptr; - qpoints->pReadBarrierMarkReg17 = is_marking ? art_quick_read_barrier_mark_reg17 : nullptr; - qpoints->pReadBarrierMarkReg18 = is_marking ? art_quick_read_barrier_mark_reg18 : nullptr; - qpoints->pReadBarrierMarkReg19 = is_marking ? art_quick_read_barrier_mark_reg19 : nullptr; - qpoints->pReadBarrierMarkReg20 = is_marking ? art_quick_read_barrier_mark_reg20 : nullptr; - qpoints->pReadBarrierMarkReg21 = is_marking ? art_quick_read_barrier_mark_reg21 : nullptr; - qpoints->pReadBarrierMarkReg22 = is_marking ? art_quick_read_barrier_mark_reg22 : nullptr; - qpoints->pReadBarrierMarkReg23 = is_marking ? art_quick_read_barrier_mark_reg23 : nullptr; - qpoints->pReadBarrierMarkReg24 = is_marking ? art_quick_read_barrier_mark_reg24 : nullptr; - qpoints->pReadBarrierMarkReg25 = is_marking ? art_quick_read_barrier_mark_reg25 : nullptr; - qpoints->pReadBarrierMarkReg26 = is_marking ? art_quick_read_barrier_mark_reg26 : nullptr; - qpoints->pReadBarrierMarkReg27 = is_marking ? art_quick_read_barrier_mark_reg27 : nullptr; - qpoints->pReadBarrierMarkReg28 = is_marking ? art_quick_read_barrier_mark_reg28 : nullptr; - qpoints->pReadBarrierMarkReg29 = is_marking ? art_quick_read_barrier_mark_reg29 : nullptr; + qpoints->pReadBarrierMarkReg00 = is_active ? art_quick_read_barrier_mark_reg00 : nullptr; + qpoints->pReadBarrierMarkReg01 = is_active ? art_quick_read_barrier_mark_reg01 : nullptr; + qpoints->pReadBarrierMarkReg02 = is_active ? art_quick_read_barrier_mark_reg02 : nullptr; + qpoints->pReadBarrierMarkReg03 = is_active ? art_quick_read_barrier_mark_reg03 : nullptr; + qpoints->pReadBarrierMarkReg04 = is_active ? art_quick_read_barrier_mark_reg04 : nullptr; + qpoints->pReadBarrierMarkReg05 = is_active ? art_quick_read_barrier_mark_reg05 : nullptr; + qpoints->pReadBarrierMarkReg06 = is_active ? art_quick_read_barrier_mark_reg06 : nullptr; + qpoints->pReadBarrierMarkReg07 = is_active ? art_quick_read_barrier_mark_reg07 : nullptr; + qpoints->pReadBarrierMarkReg08 = is_active ? art_quick_read_barrier_mark_reg08 : nullptr; + qpoints->pReadBarrierMarkReg09 = is_active ? art_quick_read_barrier_mark_reg09 : nullptr; + qpoints->pReadBarrierMarkReg10 = is_active ? art_quick_read_barrier_mark_reg10 : nullptr; + qpoints->pReadBarrierMarkReg11 = is_active ? art_quick_read_barrier_mark_reg11 : nullptr; + qpoints->pReadBarrierMarkReg12 = is_active ? art_quick_read_barrier_mark_reg12 : nullptr; + qpoints->pReadBarrierMarkReg13 = is_active ? art_quick_read_barrier_mark_reg13 : nullptr; + qpoints->pReadBarrierMarkReg14 = is_active ? art_quick_read_barrier_mark_reg14 : nullptr; + qpoints->pReadBarrierMarkReg15 = is_active ? art_quick_read_barrier_mark_reg15 : nullptr; + qpoints->pReadBarrierMarkReg17 = is_active ? art_quick_read_barrier_mark_reg17 : nullptr; + qpoints->pReadBarrierMarkReg18 = is_active ? art_quick_read_barrier_mark_reg18 : nullptr; + qpoints->pReadBarrierMarkReg19 = is_active ? art_quick_read_barrier_mark_reg19 : nullptr; + qpoints->pReadBarrierMarkReg20 = is_active ? art_quick_read_barrier_mark_reg20 : nullptr; + qpoints->pReadBarrierMarkReg21 = is_active ? art_quick_read_barrier_mark_reg21 : nullptr; + qpoints->pReadBarrierMarkReg22 = is_active ? art_quick_read_barrier_mark_reg22 : nullptr; + qpoints->pReadBarrierMarkReg23 = is_active ? art_quick_read_barrier_mark_reg23 : nullptr; + qpoints->pReadBarrierMarkReg24 = is_active ? art_quick_read_barrier_mark_reg24 : nullptr; + qpoints->pReadBarrierMarkReg25 = is_active ? art_quick_read_barrier_mark_reg25 : nullptr; + qpoints->pReadBarrierMarkReg26 = is_active ? art_quick_read_barrier_mark_reg26 : nullptr; + qpoints->pReadBarrierMarkReg27 = is_active ? art_quick_read_barrier_mark_reg27 : nullptr; + qpoints->pReadBarrierMarkReg28 = is_active ? art_quick_read_barrier_mark_reg28 : nullptr; + qpoints->pReadBarrierMarkReg29 = is_active ? art_quick_read_barrier_mark_reg29 : nullptr; // Check that array switch cases are at appropriate offsets from the introspection entrypoint. DCHECK_ALIGNED(art_quick_read_barrier_mark_introspection, 512u); @@ -128,7 +128,7 @@ void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { DCHECK_EQ(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET, gc_roots_diff); // The register 16, i.e. IP0, is reserved, so there is no art_quick_read_barrier_mark_reg16. // We're using the entry to hold a pointer to the introspection entrypoint instead. - qpoints->pReadBarrierMarkReg16 = is_marking ? art_quick_read_barrier_mark_introspection : nullptr; + qpoints->pReadBarrierMarkReg16 = is_active ? art_quick_read_barrier_mark_introspection : nullptr; } void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { @@ -188,7 +188,7 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; qpoints->pReadBarrierMarkReg16 = nullptr; // IP0 is used as a temp by the asm stub. - UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + UpdateReadBarrierEntrypoints(qpoints, /*is_active*/ false); qpoints->pReadBarrierSlow = artReadBarrierSlow; qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow; }; diff --git a/runtime/arch/mips/entrypoints_init_mips.cc b/runtime/arch/mips/entrypoints_init_mips.cc index 434e33c42a..9978da5f74 100644 --- a/runtime/arch/mips/entrypoints_init_mips.cc +++ b/runtime/arch/mips/entrypoints_init_mips.cc @@ -86,68 +86,68 @@ extern "C" double fmod(double a, double b); // REM_DOUBLE[_2ADDR] extern "C" int64_t __divdi3(int64_t, int64_t); extern "C" int64_t __moddi3(int64_t, int64_t); -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { - qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_active) { + qpoints->pReadBarrierMarkReg01 = is_active ? art_quick_read_barrier_mark_reg01 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg01), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; + qpoints->pReadBarrierMarkReg02 = is_active ? art_quick_read_barrier_mark_reg02 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg02), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; + qpoints->pReadBarrierMarkReg03 = is_active ? art_quick_read_barrier_mark_reg03 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg03), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr; + qpoints->pReadBarrierMarkReg04 = is_active ? art_quick_read_barrier_mark_reg04 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg04), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; + qpoints->pReadBarrierMarkReg05 = is_active ? art_quick_read_barrier_mark_reg05 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg05), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; + qpoints->pReadBarrierMarkReg06 = is_active ? art_quick_read_barrier_mark_reg06 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg06), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; + qpoints->pReadBarrierMarkReg07 = is_active ? art_quick_read_barrier_mark_reg07 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg07), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr; + qpoints->pReadBarrierMarkReg08 = is_active ? art_quick_read_barrier_mark_reg08 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg08), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr; + qpoints->pReadBarrierMarkReg09 = is_active ? art_quick_read_barrier_mark_reg09 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg09), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr; + qpoints->pReadBarrierMarkReg10 = is_active ? art_quick_read_barrier_mark_reg10 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg10), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr; + qpoints->pReadBarrierMarkReg11 = is_active ? art_quick_read_barrier_mark_reg11 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg11), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr; + qpoints->pReadBarrierMarkReg12 = is_active ? art_quick_read_barrier_mark_reg12 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg12), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr; + qpoints->pReadBarrierMarkReg13 = is_active ? art_quick_read_barrier_mark_reg13 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg13), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr; + qpoints->pReadBarrierMarkReg14 = is_active ? art_quick_read_barrier_mark_reg14 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg14), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg17 = is_marking ? art_quick_read_barrier_mark_reg17 : nullptr; + qpoints->pReadBarrierMarkReg17 = is_active ? art_quick_read_barrier_mark_reg17 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg17), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg18 = is_marking ? art_quick_read_barrier_mark_reg18 : nullptr; + qpoints->pReadBarrierMarkReg18 = is_active ? art_quick_read_barrier_mark_reg18 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg18), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg19 = is_marking ? art_quick_read_barrier_mark_reg19 : nullptr; + qpoints->pReadBarrierMarkReg19 = is_active ? art_quick_read_barrier_mark_reg19 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg19), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg20 = is_marking ? art_quick_read_barrier_mark_reg20 : nullptr; + qpoints->pReadBarrierMarkReg20 = is_active ? art_quick_read_barrier_mark_reg20 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg20), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg21 = is_marking ? art_quick_read_barrier_mark_reg21 : nullptr; + qpoints->pReadBarrierMarkReg21 = is_active ? art_quick_read_barrier_mark_reg21 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg21), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg22 = is_marking ? art_quick_read_barrier_mark_reg22 : nullptr; + qpoints->pReadBarrierMarkReg22 = is_active ? art_quick_read_barrier_mark_reg22 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg22), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg29 = is_marking ? art_quick_read_barrier_mark_reg29 : nullptr; + qpoints->pReadBarrierMarkReg29 = is_active ? art_quick_read_barrier_mark_reg29 : nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg29), "Non-direct C stub marked direct."); } @@ -160,7 +160,7 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { jpoints->pDlsymLookup = art_jni_dlsym_lookup_stub; // Alloc - ResetQuickAllocEntryPoints(qpoints, /*is_marking*/ false); + ResetQuickAllocEntryPoints(qpoints, /*is_active*/ false); // Cast qpoints->pInstanceofNonTrivial = artInstanceOfFromCode; @@ -412,7 +412,7 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; static_assert(IsDirectEntrypoint(kQuickReadBarrierJni), "Direct C stub not marked direct."); - UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + UpdateReadBarrierEntrypoints(qpoints, /*is_active*/ false); // Cannot use the following registers to pass arguments: // 0(ZERO), 1(AT), 16(S0), 17(S1), 24(T8), 25(T9), 26(K0), 27(K1), 28(GP), 29(SP), 31(RA). // Note that there are 30 entry points only: 00 for register 1(AT), ..., 29 for register 30(S8). diff --git a/runtime/arch/mips/quick_entrypoints_mips.S b/runtime/arch/mips/quick_entrypoints_mips.S index 61a3a04708..7bbcbf096f 100644 --- a/runtime/arch/mips/quick_entrypoints_mips.S +++ b/runtime/arch/mips/quick_entrypoints_mips.S @@ -421,7 +421,7 @@ SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP .endm -.macro RESTORE_SAVE_EVERYTHING_FRAME +.macro RESTORE_SAVE_EVERYTHING_FRAME restore_a0=1 addiu $sp, $sp, ARG_SLOT_SIZE # remove argument slots on the stack .cfi_adjust_cfa_offset -ARG_SLOT_SIZE @@ -490,8 +490,10 @@ .cfi_restore 6 lw $a1, 160($sp) .cfi_restore 5 + .if \restore_a0 lw $a0, 156($sp) .cfi_restore 4 + .endif lw $v1, 152($sp) .cfi_restore 3 lw $v0, 148($sp) @@ -507,16 +509,26 @@ .endm /* - * Macro that set calls through to artDeliverPendingExceptionFromCode, where the pending - * exception is Thread::Current()->exception_ + * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending + * exception is Thread::Current()->exception_ when the runtime method frame is ready. + * Requires $gp properly set up. */ -.macro DELIVER_PENDING_EXCEPTION - SETUP_SAVE_ALL_CALLEE_SAVES_FRAME # save callee saves for throw +.macro DELIVER_PENDING_EXCEPTION_FRAME_READY la $t9, artDeliverPendingExceptionFromCode jalr $zero, $t9 # artDeliverPendingExceptionFromCode(Thread*) move $a0, rSELF # pass Thread::Current .endm + /* + * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending + * exception is Thread::Current()->exception_. + * Requires $gp properly set up. + */ +.macro DELIVER_PENDING_EXCEPTION + SETUP_SAVE_ALL_CALLEE_SAVES_FRAME # save callee saves for throw + DELIVER_PENDING_EXCEPTION_FRAME_READY +.endm + .macro RETURN_IF_NO_EXCEPTION lw $t0, THREAD_EXCEPTION_OFFSET(rSELF) # load Thread::Current()->exception_ RESTORE_SAVE_REFS_ONLY_FRAME @@ -1660,30 +1672,51 @@ ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, art GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) +// Macro for string and type resolution and initialization. +// $a0 is both input and output. +.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint + .extern \entrypoint +ENTRY_NO_GP \name + SETUP_SAVE_EVERYTHING_FRAME # Save everything in case of GC. + move $s2, $gp # Preserve $gp across the call for exception delivery. + la $t9, \entrypoint + jalr $t9 # (uint32_t index, Thread*) + move $a1, rSELF # Pass Thread::Current (in delay slot). + beqz $v0, 1f # Success? + move $a0, $v0 # Move result to $a0 (in delay slot). + RESTORE_SAVE_EVERYTHING_FRAME 0 # Restore everything except $a0. + jalr $zero, $ra # Return on success. + nop +1: + move $gp, $s2 + DELIVER_PENDING_EXCEPTION_FRAME_READY +END \name +.endm + /* * Entry from managed code to resolve a string, this stub will allocate a String and deliver an * exception on error. On success the String is returned. A0 holds the string index. The fast * path check for hit in strings cache has already been performed. */ -ONE_ARG_DOWNCALL art_quick_resolve_string, artResolveStringFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode /* * Entry from managed code when uninitialized static storage, this stub will run the class * initializer and deliver the exception on error. On success the static storage base is * returned. */ -ONE_ARG_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode /* * Entry from managed code when dex cache misses for a type_idx. */ -ONE_ARG_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode /* * Entry from managed code when type_idx needs to be checked for access and dex cache may also * miss. */ -ONE_ARG_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode /* * Called by managed code when the value in rSUSPEND has been decremented to 0. @@ -1854,7 +1887,8 @@ ENTRY art_quick_generic_jni_trampoline nop 2: - lw $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) + lw $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) + move $gp, $s3 # restore $gp from $s3 # This will create a new save-all frame, required by the runtime. DELIVER_PENDING_EXCEPTION END art_quick_generic_jni_trampoline @@ -2213,8 +2247,32 @@ END art_quick_string_compareto */ .macro READ_BARRIER_MARK_REG name, reg ENTRY \name - /* TODO: optimizations: mark bit, forwarding. */ - addiu $sp, $sp, -160 # includes 16 bytes of space for argument registers a0-a3 + // Null check so that we can load the lock word. + bnez \reg, .Lnot_null_\name + nop +.Lret_rb_\name: + jalr $zero, $ra + nop +.Lnot_null_\name: + // Check lock word for mark bit, if marked return. + lw $t9, MIRROR_OBJECT_LOCK_WORD_OFFSET(\reg) + .set push + .set noat + sll $at, $t9, 31 - LOCK_WORD_MARK_BIT_SHIFT # Move mark bit to sign bit. + bltz $at, .Lret_rb_\name +#if (LOCK_WORD_STATE_SHIFT != 30) || (LOCK_WORD_STATE_FORWARDING_ADDRESS != 3) + // The below code depends on the lock word state being in the highest bits + // and the "forwarding address" state having all bits set. +#error "Unexpected lock word state shift or forwarding address state value." +#endif + // Test that both the forwarding state bits are 1. + sll $at, $t9, 1 + and $at, $at, $t9 # Sign bit = 1 IFF both bits are 1. + bltz $at, .Lret_forwarding_address\name + nop + .set pop + + addiu $sp, $sp, -160 # Includes 16 bytes of space for argument registers a0-a3. .cfi_adjust_cfa_offset 160 sw $ra, 156($sp) @@ -2319,6 +2377,12 @@ ENTRY \name jalr $zero, $ra addiu $sp, $sp, 160 .cfi_adjust_cfa_offset -160 + +.Lret_forwarding_address\name: + jalr $zero, $ra + // Shift left by the forwarding address shift. This clears out the state bits since they are + // in the top 2 bits of the lock word. + sll \reg, $t9, LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT END \name .endm diff --git a/runtime/arch/mips64/entrypoints_init_mips64.cc b/runtime/arch/mips64/entrypoints_init_mips64.cc index f8242ae1b5..763d93eb47 100644 --- a/runtime/arch/mips64/entrypoints_init_mips64.cc +++ b/runtime/arch/mips64/entrypoints_init_mips64.cc @@ -86,27 +86,27 @@ extern "C" int64_t __divdi3(int64_t, int64_t); extern "C" int64_t __moddi3(int64_t, int64_t); // No read barrier entrypoints for marking registers. -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { - qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; - qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; - qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; - qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr; - qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; - qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; - qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; - qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr; - qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr; - qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr; - qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr; - qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr; - qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr; - qpoints->pReadBarrierMarkReg17 = is_marking ? art_quick_read_barrier_mark_reg17 : nullptr; - qpoints->pReadBarrierMarkReg18 = is_marking ? art_quick_read_barrier_mark_reg18 : nullptr; - qpoints->pReadBarrierMarkReg19 = is_marking ? art_quick_read_barrier_mark_reg19 : nullptr; - qpoints->pReadBarrierMarkReg20 = is_marking ? art_quick_read_barrier_mark_reg20 : nullptr; - qpoints->pReadBarrierMarkReg21 = is_marking ? art_quick_read_barrier_mark_reg21 : nullptr; - qpoints->pReadBarrierMarkReg22 = is_marking ? art_quick_read_barrier_mark_reg22 : nullptr; - qpoints->pReadBarrierMarkReg29 = is_marking ? art_quick_read_barrier_mark_reg29 : nullptr; +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_active) { + qpoints->pReadBarrierMarkReg01 = is_active ? art_quick_read_barrier_mark_reg01 : nullptr; + qpoints->pReadBarrierMarkReg02 = is_active ? art_quick_read_barrier_mark_reg02 : nullptr; + qpoints->pReadBarrierMarkReg03 = is_active ? art_quick_read_barrier_mark_reg03 : nullptr; + qpoints->pReadBarrierMarkReg04 = is_active ? art_quick_read_barrier_mark_reg04 : nullptr; + qpoints->pReadBarrierMarkReg05 = is_active ? art_quick_read_barrier_mark_reg05 : nullptr; + qpoints->pReadBarrierMarkReg06 = is_active ? art_quick_read_barrier_mark_reg06 : nullptr; + qpoints->pReadBarrierMarkReg07 = is_active ? art_quick_read_barrier_mark_reg07 : nullptr; + qpoints->pReadBarrierMarkReg08 = is_active ? art_quick_read_barrier_mark_reg08 : nullptr; + qpoints->pReadBarrierMarkReg09 = is_active ? art_quick_read_barrier_mark_reg09 : nullptr; + qpoints->pReadBarrierMarkReg10 = is_active ? art_quick_read_barrier_mark_reg10 : nullptr; + qpoints->pReadBarrierMarkReg11 = is_active ? art_quick_read_barrier_mark_reg11 : nullptr; + qpoints->pReadBarrierMarkReg12 = is_active ? art_quick_read_barrier_mark_reg12 : nullptr; + qpoints->pReadBarrierMarkReg13 = is_active ? art_quick_read_barrier_mark_reg13 : nullptr; + qpoints->pReadBarrierMarkReg17 = is_active ? art_quick_read_barrier_mark_reg17 : nullptr; + qpoints->pReadBarrierMarkReg18 = is_active ? art_quick_read_barrier_mark_reg18 : nullptr; + qpoints->pReadBarrierMarkReg19 = is_active ? art_quick_read_barrier_mark_reg19 : nullptr; + qpoints->pReadBarrierMarkReg20 = is_active ? art_quick_read_barrier_mark_reg20 : nullptr; + qpoints->pReadBarrierMarkReg21 = is_active ? art_quick_read_barrier_mark_reg21 : nullptr; + qpoints->pReadBarrierMarkReg22 = is_active ? art_quick_read_barrier_mark_reg22 : nullptr; + qpoints->pReadBarrierMarkReg29 = is_active ? art_quick_read_barrier_mark_reg29 : nullptr; } void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { @@ -168,7 +168,7 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + UpdateReadBarrierEntrypoints(qpoints, /*is_active*/ false); // Cannot use the following registers to pass arguments: // 0(ZERO), 1(AT), 15(T3), 16(S0), 17(S1), 24(T8), 25(T9), 26(K0), 27(K1), 28(GP), 29(SP), 31(RA). // Note that there are 30 entry points only: 00 for register 1(AT), ..., 29 for register 30(S8). diff --git a/runtime/arch/mips64/quick_entrypoints_mips64.S b/runtime/arch/mips64/quick_entrypoints_mips64.S index 24caa0e290..8f713a1928 100644 --- a/runtime/arch/mips64/quick_entrypoints_mips64.S +++ b/runtime/arch/mips64/quick_entrypoints_mips64.S @@ -447,7 +447,7 @@ SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP .endm -.macro RESTORE_SAVE_EVERYTHING_FRAME +.macro RESTORE_SAVE_EVERYTHING_FRAME restore_a0=1 // Restore FP registers. l.d $f31, 264($sp) l.d $f30, 256($sp) @@ -530,8 +530,10 @@ .cfi_restore 6 ld $a1, 304($sp) .cfi_restore 5 + .if \restore_a0 ld $a0, 296($sp) .cfi_restore 4 + .endif ld $v1, 288($sp) .cfi_restore 3 ld $v0, 280($sp) @@ -547,18 +549,26 @@ .endm /* - * Macro that set calls through to artDeliverPendingExceptionFromCode, - * where the pending - * exception is Thread::Current()->exception_ + * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending + * exception is Thread::Current()->exception_ when the runtime method frame is ready. + * Requires $gp properly set up. */ -.macro DELIVER_PENDING_EXCEPTION - SETUP_GP - SETUP_SAVE_ALL_CALLEE_SAVES_FRAME # save callee saves for throw +.macro DELIVER_PENDING_EXCEPTION_FRAME_READY dla $t9, artDeliverPendingExceptionFromCode jalr $zero, $t9 # artDeliverPendingExceptionFromCode(Thread*) move $a0, rSELF # pass Thread::Current .endm + /* + * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending + * exception is Thread::Current()->exception_. + */ +.macro DELIVER_PENDING_EXCEPTION + SETUP_GP + SETUP_SAVE_ALL_CALLEE_SAVES_FRAME # save callee saves for throw + DELIVER_PENDING_EXCEPTION_FRAME_READY +.endm + .macro RETURN_IF_NO_EXCEPTION ld $t0, THREAD_EXCEPTION_OFFSET(rSELF) # load Thread::Current()->exception_ RESTORE_SAVE_REFS_ONLY_FRAME @@ -1615,30 +1625,48 @@ ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, art GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) +// Macro for string and type resolution and initialization. +// $a0 is both input and output. +.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint + .extern \entrypoint +ENTRY_NO_GP \name + SETUP_SAVE_EVERYTHING_FRAME # Save everything in case of GC. + dla $t9, \entrypoint + jalr $t9 # (uint32_t index, Thread*) + move $a1, rSELF # Pass Thread::Current (in delay slot). + beqz $v0, 1f # Success? + move $a0, $v0 # Move result to $a0 (in delay slot). + RESTORE_SAVE_EVERYTHING_FRAME 0 # Restore everything except $a0. + jic $ra, 0 # Return on success. +1: + DELIVER_PENDING_EXCEPTION_FRAME_READY +END \name +.endm + /* * Entry from managed code to resolve a string, this stub will allocate a String and deliver an * exception on error. On success the String is returned. A0 holds the string index. The fast * path check for hit in strings cache has already been performed. */ -ONE_ARG_DOWNCALL art_quick_resolve_string, artResolveStringFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode /* * Entry from managed code when uninitialized static storage, this stub will run the class * initializer and deliver the exception on error. On success the static storage base is * returned. */ -ONE_ARG_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode /* * Entry from managed code when dex cache misses for a type_idx. */ -ONE_ARG_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode /* * Entry from managed code when type_idx needs to be checked for access and dex cache may also * miss. */ -ONE_ARG_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode /* * Called by managed code when the value in rSUSPEND has been decremented to 0. @@ -2067,7 +2095,29 @@ END art_quick_indexof */ .macro READ_BARRIER_MARK_REG name, reg ENTRY \name - /* TODO: optimizations: mark bit, forwarding. */ + // Null check so that we can load the lock word. + bnezc \reg, .Lnot_null_\name + nop +.Lret_rb_\name: + jic $ra, 0 +.Lnot_null_\name: + // Check lock word for mark bit, if marked return. + lw $t9, MIRROR_OBJECT_LOCK_WORD_OFFSET(\reg) + .set push + .set noat + sll $at, $t9, 31 - LOCK_WORD_MARK_BIT_SHIFT # Move mark bit to sign bit. + bltzc $at, .Lret_rb_\name +#if (LOCK_WORD_STATE_SHIFT != 30) || (LOCK_WORD_STATE_FORWARDING_ADDRESS != 3) + // The below code depends on the lock word state being in the highest bits + // and the "forwarding address" state having all bits set. +#error "Unexpected lock word state shift or forwarding address state value." +#endif + // Test that both the forwarding state bits are 1. + sll $at, $t9, 1 + and $at, $at, $t9 # Sign bit = 1 IFF both bits are 1. + bltzc $at, .Lret_forwarding_address\name + .set pop + daddiu $sp, $sp, -320 .cfi_adjust_cfa_offset 320 @@ -2202,6 +2252,13 @@ ENTRY \name jalr $zero, $ra daddiu $sp, $sp, 320 .cfi_adjust_cfa_offset -320 + +.Lret_forwarding_address\name: + // Shift left by the forwarding address shift. This clears out the state bits since they are + // in the top 2 bits of the lock word. + sll \reg, $t9, LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT + jalr $zero, $ra + dext \reg, \reg, 0, 32 # Make sure the address is zero-extended. END \name .endm diff --git a/runtime/arch/x86/entrypoints_init_x86.cc b/runtime/arch/x86/entrypoints_init_x86.cc index 9cd4a3ee3b..102faf19d4 100644 --- a/runtime/arch/x86/entrypoints_init_x86.cc +++ b/runtime/arch/x86/entrypoints_init_x86.cc @@ -44,14 +44,14 @@ extern "C" mirror::Object* art_quick_read_barrier_mark_reg07(mirror::Object*); extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t); extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*); -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { - qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr; - qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; - qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; - qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; - qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; - qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; - qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_active) { + qpoints->pReadBarrierMarkReg00 = is_active ? art_quick_read_barrier_mark_reg00 : nullptr; + qpoints->pReadBarrierMarkReg01 = is_active ? art_quick_read_barrier_mark_reg01 : nullptr; + qpoints->pReadBarrierMarkReg02 = is_active ? art_quick_read_barrier_mark_reg02 : nullptr; + qpoints->pReadBarrierMarkReg03 = is_active ? art_quick_read_barrier_mark_reg03 : nullptr; + qpoints->pReadBarrierMarkReg05 = is_active ? art_quick_read_barrier_mark_reg05 : nullptr; + qpoints->pReadBarrierMarkReg06 = is_active ? art_quick_read_barrier_mark_reg06 : nullptr; + qpoints->pReadBarrierMarkReg07 = is_active ? art_quick_read_barrier_mark_reg07 : nullptr; } void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { @@ -97,7 +97,7 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + UpdateReadBarrierEntrypoints(qpoints, /*is_active*/ false); qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (ESP) to pass arguments. // x86 has only 8 core registers. qpoints->pReadBarrierMarkReg08 = nullptr; diff --git a/runtime/arch/x86_64/entrypoints_init_x86_64.cc b/runtime/arch/x86_64/entrypoints_init_x86_64.cc index a326b4eebc..1e56e8a087 100644 --- a/runtime/arch/x86_64/entrypoints_init_x86_64.cc +++ b/runtime/arch/x86_64/entrypoints_init_x86_64.cc @@ -55,22 +55,22 @@ extern "C" mirror::Object* art_quick_read_barrier_mark_reg15(mirror::Object*); extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t); extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*); -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { - qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr; - qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; - qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; - qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; - qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; - qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; - qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; - qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr; - qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr; - qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr; - qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr; - qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr; - qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr; - qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr; - qpoints->pReadBarrierMarkReg15 = is_marking ? art_quick_read_barrier_mark_reg15 : nullptr; +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_active) { + qpoints->pReadBarrierMarkReg00 = is_active ? art_quick_read_barrier_mark_reg00 : nullptr; + qpoints->pReadBarrierMarkReg01 = is_active ? art_quick_read_barrier_mark_reg01 : nullptr; + qpoints->pReadBarrierMarkReg02 = is_active ? art_quick_read_barrier_mark_reg02 : nullptr; + qpoints->pReadBarrierMarkReg03 = is_active ? art_quick_read_barrier_mark_reg03 : nullptr; + qpoints->pReadBarrierMarkReg05 = is_active ? art_quick_read_barrier_mark_reg05 : nullptr; + qpoints->pReadBarrierMarkReg06 = is_active ? art_quick_read_barrier_mark_reg06 : nullptr; + qpoints->pReadBarrierMarkReg07 = is_active ? art_quick_read_barrier_mark_reg07 : nullptr; + qpoints->pReadBarrierMarkReg08 = is_active ? art_quick_read_barrier_mark_reg08 : nullptr; + qpoints->pReadBarrierMarkReg09 = is_active ? art_quick_read_barrier_mark_reg09 : nullptr; + qpoints->pReadBarrierMarkReg10 = is_active ? art_quick_read_barrier_mark_reg10 : nullptr; + qpoints->pReadBarrierMarkReg11 = is_active ? art_quick_read_barrier_mark_reg11 : nullptr; + qpoints->pReadBarrierMarkReg12 = is_active ? art_quick_read_barrier_mark_reg12 : nullptr; + qpoints->pReadBarrierMarkReg13 = is_active ? art_quick_read_barrier_mark_reg13 : nullptr; + qpoints->pReadBarrierMarkReg14 = is_active ? art_quick_read_barrier_mark_reg14 : nullptr; + qpoints->pReadBarrierMarkReg15 = is_active ? art_quick_read_barrier_mark_reg15 : nullptr; } void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { @@ -119,7 +119,7 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + UpdateReadBarrierEntrypoints(qpoints, /*is_active*/ false); qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (RSP) to pass arguments. // x86-64 has only 16 core registers. qpoints->pReadBarrierMarkReg16 = nullptr; diff --git a/runtime/base/mutex.h b/runtime/base/mutex.h index 2414b5f937..03ae63a068 100644 --- a/runtime/base/mutex.h +++ b/runtime/base/mutex.h @@ -373,19 +373,19 @@ class SHARED_LOCKABLE ReaderWriterMutex : public BaseMutex { bool IsSharedHeld(const Thread* self) const; // Assert the current thread has shared access to the ReaderWriterMutex. - void AssertSharedHeld(const Thread* self) ASSERT_SHARED_CAPABILITY(this) { + ALWAYS_INLINE void AssertSharedHeld(const Thread* self) ASSERT_SHARED_CAPABILITY(this) { if (kDebugLocking && (gAborting == 0)) { // TODO: we can only assert this well when self != null. CHECK(IsSharedHeld(self) || self == nullptr) << *this; } } - void AssertReaderHeld(const Thread* self) ASSERT_SHARED_CAPABILITY(this) { + ALWAYS_INLINE void AssertReaderHeld(const Thread* self) ASSERT_SHARED_CAPABILITY(this) { AssertSharedHeld(self); } // Assert the current thread doesn't hold this ReaderWriterMutex either in shared or exclusive // mode. - void AssertNotHeld(const Thread* self) ASSERT_SHARED_CAPABILITY(!this) { + ALWAYS_INLINE void AssertNotHeld(const Thread* self) ASSERT_SHARED_CAPABILITY(!this) { if (kDebugLocking && (gAborting == 0)) { CHECK(!IsSharedHeld(self)) << *this; } diff --git a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc index 355d7b3e2f..6b965678c3 100644 --- a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc @@ -58,18 +58,13 @@ static inline void BssWriteBarrier(ArtMethod* outer_method) REQUIRES_SHARED(Lock } } -constexpr Runtime::CalleeSaveType kInitEntrypointSaveType = - // TODO: Change allocation entrypoints on MIPS and MIPS64 to kSaveEverything. - (kRuntimeISA == kMips || kRuntimeISA == kMips64) ? Runtime::kSaveRefsOnly - : Runtime::kSaveEverything; - extern "C" mirror::Class* artInitializeStaticStorageFromCode(uint32_t type_idx, Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) { // Called to ensure static storage base is initialized for direct static field reads and writes. // A class may be accessing another class' fields when it doesn't have access, as access has been // given by inheritance. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, kInitEntrypointSaveType); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, true, false); @@ -83,7 +78,7 @@ extern "C" mirror::Class* artInitializeTypeFromCode(uint32_t type_idx, Thread* s REQUIRES_SHARED(Locks::mutator_lock_) { // Called when method->dex_cache_resolved_types_[] misses. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, kInitEntrypointSaveType); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, false, false); @@ -98,7 +93,7 @@ extern "C" mirror::Class* artInitializeTypeAndVerifyAccessFromCode(uint32_t type // Called when caller isn't guaranteed to have access to a type and the dex cache may be // unpopulated. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, kInitEntrypointSaveType); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, false, true); @@ -111,7 +106,7 @@ extern "C" mirror::Class* artInitializeTypeAndVerifyAccessFromCode(uint32_t type extern "C" mirror::String* artResolveStringFromCode(int32_t string_idx, Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, kInitEntrypointSaveType); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::String* result = ResolveStringFromCode(caller, dex::StringIndex(string_idx)); if (LIKELY(result != nullptr)) { diff --git a/runtime/gc/accounting/card_table.h b/runtime/gc/accounting/card_table.h index cd30d9d149..c3dd21f113 100644 --- a/runtime/gc/accounting/card_table.h +++ b/runtime/gc/accounting/card_table.h @@ -51,6 +51,7 @@ class CardTable { static constexpr size_t kCardSize = 1 << kCardShift; static constexpr uint8_t kCardClean = 0x0; static constexpr uint8_t kCardDirty = 0x70; + static constexpr uint8_t kCardAged = kCardDirty - 1; static CardTable* Create(const uint8_t* heap_begin, size_t heap_capacity); ~CardTable(); diff --git a/runtime/gc/collector/concurrent_copying-inl.h b/runtime/gc/collector/concurrent_copying-inl.h index d5c36bfb19..3503973321 100644 --- a/runtime/gc/collector/concurrent_copying-inl.h +++ b/runtime/gc/collector/concurrent_copying-inl.h @@ -152,7 +152,8 @@ inline mirror::Object* ConcurrentCopying::Mark(mirror::Object* from_ref, inline mirror::Object* ConcurrentCopying::MarkFromReadBarrier(mirror::Object* from_ref) { mirror::Object* ret; - if (from_ref == nullptr) { + // We can get here before marking starts since we gray immune objects before the marking phase. + if (from_ref == nullptr || !Thread::Current()->GetIsGcMarking()) { return from_ref; } // TODO: Consider removing this check when we are done investigating slow paths. b/30162165 diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index e27c1ecb08..a450a751b8 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -77,6 +77,7 @@ ConcurrentCopying::ConcurrentCopying(Heap* heap, mark_stack_lock_("concurrent copying mark stack lock", kMarkSweepMarkStackLock), thread_running_gc_(nullptr), is_marking_(false), + is_using_read_barrier_entrypoints_(false), is_active_(false), is_asserting_to_space_invariant_(false), region_space_bitmap_(nullptr), @@ -163,6 +164,15 @@ void ConcurrentCopying::RunPhases() { ReaderMutexLock mu(self, *Locks::mutator_lock_); InitializePhase(); } + if (kUseBakerReadBarrier && kGrayDirtyImmuneObjects) { + // Switch to read barrier mark entrypoints before we gray the objects. This is required in case + // a mutator sees a gray bit and dispatches on the entrpoint. (b/37876887). + ActivateReadBarrierEntrypoints(); + // Gray dirty immune objects concurrently to reduce GC pause times. We re-process gray cards in + // the pause. + ReaderMutexLock mu(self, *Locks::mutator_lock_); + GrayAllDirtyImmuneObjects(); + } FlipThreadRoots(); { ReaderMutexLock mu(self, *Locks::mutator_lock_); @@ -192,6 +202,59 @@ void ConcurrentCopying::RunPhases() { thread_running_gc_ = nullptr; } +class ConcurrentCopying::ActivateReadBarrierEntrypointsCheckpoint : public Closure { + public: + explicit ActivateReadBarrierEntrypointsCheckpoint(ConcurrentCopying* concurrent_copying) + : concurrent_copying_(concurrent_copying) {} + + void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { + // Note: self is not necessarily equal to thread since thread may be suspended. + Thread* self = Thread::Current(); + DCHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) + << thread->GetState() << " thread " << thread << " self " << self; + // Switch to the read barrier entrypoints. + thread->SetReadBarrierEntrypoints(); + // If thread is a running mutator, then act on behalf of the garbage collector. + // See the code in ThreadList::RunCheckpoint. + concurrent_copying_->GetBarrier().Pass(self); + } + + private: + ConcurrentCopying* const concurrent_copying_; +}; + +class ConcurrentCopying::ActivateReadBarrierEntrypointsCallback : public Closure { + public: + explicit ActivateReadBarrierEntrypointsCallback(ConcurrentCopying* concurrent_copying) + : concurrent_copying_(concurrent_copying) {} + + void Run(Thread* self ATTRIBUTE_UNUSED) OVERRIDE REQUIRES(Locks::thread_list_lock_) { + // This needs to run under the thread_list_lock_ critical section in ThreadList::RunCheckpoint() + // to avoid a race with ThreadList::Register(). + CHECK(!concurrent_copying_->is_using_read_barrier_entrypoints_); + concurrent_copying_->is_using_read_barrier_entrypoints_ = true; + } + + private: + ConcurrentCopying* const concurrent_copying_; +}; + +void ConcurrentCopying::ActivateReadBarrierEntrypoints() { + Thread* const self = Thread::Current(); + ActivateReadBarrierEntrypointsCheckpoint checkpoint(this); + ThreadList* thread_list = Runtime::Current()->GetThreadList(); + gc_barrier_->Init(self, 0); + ActivateReadBarrierEntrypointsCallback callback(this); + const size_t barrier_count = thread_list->RunCheckpoint(&checkpoint, &callback); + // If there are no threads to wait which implies that all the checkpoint functions are finished, + // then no need to release the mutator lock. + if (barrier_count == 0) { + return; + } + ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); + gc_barrier_->Increment(self, barrier_count); +} + void ConcurrentCopying::BindBitmaps() { Thread* self = Thread::Current(); WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); @@ -352,9 +415,12 @@ class ConcurrentCopying::FlipCallback : public Closure { if (kVerifyNoMissingCardMarks) { cc->VerifyNoMissingCardMarks(); } - CHECK(thread == self); + CHECK_EQ(thread, self); Locks::mutator_lock_->AssertExclusiveHeld(self); - cc->region_space_->SetFromSpace(cc->rb_table_, cc->force_evacuate_all_); + { + TimingLogger::ScopedTiming split2("(Paused)SetFromSpace", cc->GetTimings()); + cc->region_space_->SetFromSpace(cc->rb_table_, cc->force_evacuate_all_); + } cc->SwapStacks(); if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) { cc->RecordLiveStackFreezeSize(self); @@ -368,11 +434,11 @@ class ConcurrentCopying::FlipCallback : public Closure { } if (UNLIKELY(Runtime::Current()->IsActiveTransaction())) { CHECK(Runtime::Current()->IsAotCompiler()); - TimingLogger::ScopedTiming split2("(Paused)VisitTransactionRoots", cc->GetTimings()); + TimingLogger::ScopedTiming split3("(Paused)VisitTransactionRoots", cc->GetTimings()); Runtime::Current()->VisitTransactionRoots(cc); } if (kUseBakerReadBarrier && kGrayDirtyImmuneObjects) { - cc->GrayAllDirtyImmuneObjects(); + cc->GrayAllNewlyDirtyImmuneObjects(); if (kIsDebugBuild) { // Check that all non-gray immune objects only refernce immune objects. cc->VerifyGrayImmuneObjects(); @@ -519,8 +585,8 @@ class ConcurrentCopying::VerifyNoMissingCardMarkVisitor { void ConcurrentCopying::VerifyNoMissingCardMarkCallback(mirror::Object* obj, void* arg) { auto* collector = reinterpret_cast<ConcurrentCopying*>(arg); - // Objects not on dirty cards should never have references to newly allocated regions. - if (!collector->heap_->GetCardTable()->IsDirty(obj)) { + // Objects not on dirty or aged cards should never have references to newly allocated regions. + if (collector->heap_->GetCardTable()->GetCard(obj) == gc::accounting::CardTable::kCardClean) { VerifyNoMissingCardMarkVisitor visitor(collector, /*holder*/ obj); obj->VisitReferences</*kVisitNativeRoots*/true, kVerifyNone, kWithoutReadBarrier>( visitor, @@ -583,53 +649,100 @@ void ConcurrentCopying::FlipThreadRoots() { } } +template <bool kConcurrent> class ConcurrentCopying::GrayImmuneObjectVisitor { public: - explicit GrayImmuneObjectVisitor() {} + explicit GrayImmuneObjectVisitor(Thread* self) : self_(self) {} ALWAYS_INLINE void operator()(mirror::Object* obj) const REQUIRES_SHARED(Locks::mutator_lock_) { - if (kUseBakerReadBarrier) { - if (kIsDebugBuild) { - Locks::mutator_lock_->AssertExclusiveHeld(Thread::Current()); + if (kUseBakerReadBarrier && obj->GetReadBarrierState() == ReadBarrier::WhiteState()) { + if (kConcurrent) { + Locks::mutator_lock_->AssertSharedHeld(self_); + obj->AtomicSetReadBarrierState(ReadBarrier::WhiteState(), ReadBarrier::GrayState()); + // Mod union table VisitObjects may visit the same object multiple times so we can't check + // the result of the atomic set. + } else { + Locks::mutator_lock_->AssertExclusiveHeld(self_); + obj->SetReadBarrierState(ReadBarrier::GrayState()); } - obj->SetReadBarrierState(ReadBarrier::GrayState()); } } static void Callback(mirror::Object* obj, void* arg) REQUIRES_SHARED(Locks::mutator_lock_) { - reinterpret_cast<GrayImmuneObjectVisitor*>(arg)->operator()(obj); + reinterpret_cast<GrayImmuneObjectVisitor<kConcurrent>*>(arg)->operator()(obj); } + + private: + Thread* const self_; }; void ConcurrentCopying::GrayAllDirtyImmuneObjects() { - TimingLogger::ScopedTiming split(__FUNCTION__, GetTimings()); - gc::Heap* const heap = Runtime::Current()->GetHeap(); - accounting::CardTable* const card_table = heap->GetCardTable(); - WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); + TimingLogger::ScopedTiming split("GrayAllDirtyImmuneObjects", GetTimings()); + accounting::CardTable* const card_table = heap_->GetCardTable(); + Thread* const self = Thread::Current(); + using VisitorType = GrayImmuneObjectVisitor</* kIsConcurrent */ true>; + VisitorType visitor(self); + WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); for (space::ContinuousSpace* space : immune_spaces_.GetSpaces()) { DCHECK(space->IsImageSpace() || space->IsZygoteSpace()); - GrayImmuneObjectVisitor visitor; - accounting::ModUnionTable* table = heap->FindModUnionTableFromSpace(space); + accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space); // Mark all the objects on dirty cards since these may point to objects in other space. // Once these are marked, the GC will eventually clear them later. // Table is non null for boot image and zygote spaces. It is only null for application image // spaces. if (table != nullptr) { - // TODO: Consider adding precleaning outside the pause. table->ProcessCards(); - table->VisitObjects(GrayImmuneObjectVisitor::Callback, &visitor); - // Since the cards are recorded in the mod-union table and this is paused, we can clear - // the cards for the space (to madvise). + table->VisitObjects(&VisitorType::Callback, &visitor); + // Don't clear cards here since we need to rescan in the pause. If we cleared the cards here, + // there would be races with the mutator marking new cards. + } else { + // Keep cards aged if we don't have a mod-union table since we may need to scan them in future + // GCs. This case is for app images. + card_table->ModifyCardsAtomic( + space->Begin(), + space->End(), + [](uint8_t card) { + return (card != gc::accounting::CardTable::kCardClean) + ? gc::accounting::CardTable::kCardAged + : card; + }, + /* card modified visitor */ VoidFunctor()); + card_table->Scan</* kClearCard */ false>(space->GetMarkBitmap(), + space->Begin(), + space->End(), + visitor, + gc::accounting::CardTable::kCardAged); + } + } +} + +void ConcurrentCopying::GrayAllNewlyDirtyImmuneObjects() { + TimingLogger::ScopedTiming split("(Paused)GrayAllNewlyDirtyImmuneObjects", GetTimings()); + accounting::CardTable* const card_table = heap_->GetCardTable(); + using VisitorType = GrayImmuneObjectVisitor</* kIsConcurrent */ false>; + Thread* const self = Thread::Current(); + VisitorType visitor(self); + WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); + for (space::ContinuousSpace* space : immune_spaces_.GetSpaces()) { + DCHECK(space->IsImageSpace() || space->IsZygoteSpace()); + accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space); + + // Don't need to scan aged cards since we did these before the pause. Note that scanning cards + // also handles the mod-union table cards. + card_table->Scan</* kClearCard */ false>(space->GetMarkBitmap(), + space->Begin(), + space->End(), + visitor, + gc::accounting::CardTable::kCardDirty); + if (table != nullptr) { + // Add the cards to the mod-union table so that we can clear cards to save RAM. + table->ProcessCards(); TimingLogger::ScopedTiming split2("(Paused)ClearCards", GetTimings()); card_table->ClearCardRange(space->Begin(), AlignDown(space->End(), accounting::CardTable::kCardSize)); - } else { - // TODO: Consider having a mark bitmap for app image spaces and avoid scanning during the - // pause because app image spaces are all dirty pages anyways. - card_table->Scan<false>(space->GetMarkBitmap(), space->Begin(), space->End(), visitor); } } - // Since all of the objects that may point to other spaces are marked, we can avoid all the read + // Since all of the objects that may point to other spaces are gray, we can avoid all the read // barriers in the immune spaces. updated_all_immune_objects_.StoreRelaxed(true); } @@ -658,6 +771,7 @@ class ConcurrentCopying::ImmuneSpaceScanObjVisitor { ALWAYS_INLINE void operator()(mirror::Object* obj) const REQUIRES_SHARED(Locks::mutator_lock_) { if (kUseBakerReadBarrier && kGrayDirtyImmuneObjects) { + // Only need to scan gray objects. if (obj->GetReadBarrierState() == ReadBarrier::GrayState()) { collector_->ScanImmuneObject(obj); // Done scanning the object, go back to white. @@ -707,6 +821,7 @@ void ConcurrentCopying::MarkingPhase() { if (kUseBakerReadBarrier && kGrayDirtyImmuneObjects && table != nullptr) { table->VisitObjects(ImmuneSpaceScanObjVisitor::Callback, &visitor); } else { + // TODO: Scan only the aged cards. live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), reinterpret_cast<uintptr_t>(space->Limit()), visitor); @@ -876,6 +991,12 @@ class ConcurrentCopying::DisableMarkingCallback : public Closure { // to avoid a race with ThreadList::Register(). CHECK(concurrent_copying_->is_marking_); concurrent_copying_->is_marking_ = false; + if (kUseBakerReadBarrier && kGrayDirtyImmuneObjects) { + CHECK(concurrent_copying_->is_using_read_barrier_entrypoints_); + concurrent_copying_->is_using_read_barrier_entrypoints_ = false; + } else { + CHECK(!concurrent_copying_->is_using_read_barrier_entrypoints_); + } } private: @@ -1621,25 +1742,29 @@ void ConcurrentCopying::MarkZygoteLargeObjects() { Thread* const self = Thread::Current(); WriterMutexLock rmu(self, *Locks::heap_bitmap_lock_); space::LargeObjectSpace* const los = heap_->GetLargeObjectsSpace(); - // Pick the current live bitmap (mark bitmap if swapped). - accounting::LargeObjectBitmap* const live_bitmap = los->GetLiveBitmap(); - accounting::LargeObjectBitmap* const mark_bitmap = los->GetMarkBitmap(); - // Walk through all of the objects and explicitly mark the zygote ones so they don't get swept. - std::pair<uint8_t*, uint8_t*> range = los->GetBeginEndAtomic(); - live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(range.first), - reinterpret_cast<uintptr_t>(range.second), - [mark_bitmap, los, self](mirror::Object* obj) - REQUIRES(Locks::heap_bitmap_lock_) - REQUIRES_SHARED(Locks::mutator_lock_) { - if (los->IsZygoteLargeObject(self, obj)) { - mark_bitmap->Set(obj); - } - }); + if (los != nullptr) { + // Pick the current live bitmap (mark bitmap if swapped). + accounting::LargeObjectBitmap* const live_bitmap = los->GetLiveBitmap(); + accounting::LargeObjectBitmap* const mark_bitmap = los->GetMarkBitmap(); + // Walk through all of the objects and explicitly mark the zygote ones so they don't get swept. + std::pair<uint8_t*, uint8_t*> range = los->GetBeginEndAtomic(); + live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(range.first), + reinterpret_cast<uintptr_t>(range.second), + [mark_bitmap, los, self](mirror::Object* obj) + REQUIRES(Locks::heap_bitmap_lock_) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (los->IsZygoteLargeObject(self, obj)) { + mark_bitmap->Set(obj); + } + }); + } } void ConcurrentCopying::SweepLargeObjects(bool swap_bitmaps) { TimingLogger::ScopedTiming split("SweepLargeObjects", GetTimings()); - RecordFreeLOS(heap_->GetLargeObjectsSpace()->Sweep(swap_bitmaps)); + if (heap_->GetLargeObjectsSpace() != nullptr) { + RecordFreeLOS(heap_->GetLargeObjectsSpace()->Sweep(swap_bitmaps)); + } } void ConcurrentCopying::ReclaimPhase() { @@ -1888,7 +2013,6 @@ void ConcurrentCopying::AssertToSpaceInvariantInNonMovingSpace(mirror::Object* o heap_mark_bitmap_->GetContinuousSpaceBitmap(ref); accounting::LargeObjectBitmap* los_bitmap = heap_mark_bitmap_->GetLargeObjectBitmap(ref); - CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; bool is_los = mark_bitmap == nullptr; if ((!is_los && mark_bitmap->Test(ref)) || (is_los && los_bitmap->Test(ref))) { @@ -2392,7 +2516,6 @@ mirror::Object* ConcurrentCopying::MarkNonMoving(mirror::Object* ref, heap_mark_bitmap_->GetContinuousSpaceBitmap(ref); accounting::LargeObjectBitmap* los_bitmap = heap_mark_bitmap_->GetLargeObjectBitmap(ref); - CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; bool is_los = mark_bitmap == nullptr; if (!is_los && mark_bitmap->Test(ref)) { // Already marked. diff --git a/runtime/gc/collector/concurrent_copying.h b/runtime/gc/collector/concurrent_copying.h index 37b6a2c541..c09e0eb109 100644 --- a/runtime/gc/collector/concurrent_copying.h +++ b/runtime/gc/collector/concurrent_copying.h @@ -118,6 +118,11 @@ class ConcurrentCopying : public GarbageCollector { bool IsMarking() const { return is_marking_; } + // We may want to use read barrier entrypoints before is_marking_ is true since concurrent graying + // creates a small window where we might dispatch on these entrypoints. + bool IsUsingReadBarrierEntrypoints() const { + return is_using_read_barrier_entrypoints_; + } bool IsActive() const { return is_active_; } @@ -165,6 +170,9 @@ class ConcurrentCopying : public GarbageCollector { void GrayAllDirtyImmuneObjects() REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); + void GrayAllNewlyDirtyImmuneObjects() + REQUIRES(Locks::mutator_lock_) + REQUIRES(!mark_stack_lock_); void VerifyGrayImmuneObjects() REQUIRES(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); @@ -252,6 +260,8 @@ class ConcurrentCopying : public GarbageCollector { REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); void DumpPerformanceInfo(std::ostream& os) OVERRIDE REQUIRES(!rb_slow_path_histogram_lock_); + // Set the read barrier mark entrypoints to non-null. + void ActivateReadBarrierEntrypoints(); space::RegionSpace* region_space_; // The underlying region space. std::unique_ptr<Barrier> gc_barrier_; @@ -268,6 +278,8 @@ class ConcurrentCopying : public GarbageCollector { GUARDED_BY(mark_stack_lock_); Thread* thread_running_gc_; bool is_marking_; // True while marking is ongoing. + // True while we might dispatch on the read barrier entrypoints. + bool is_using_read_barrier_entrypoints_; bool is_active_; // True while the collection is ongoing. bool is_asserting_to_space_invariant_; // True while asserting the to-space invariant. ImmuneSpaces immune_spaces_; @@ -330,6 +342,8 @@ class ConcurrentCopying : public GarbageCollector { // ObjPtr since the GC may transition to suspended and runnable between phases. mirror::Class* java_lang_Object_; + class ActivateReadBarrierEntrypointsCallback; + class ActivateReadBarrierEntrypointsCheckpoint; class AssertToSpaceInvariantFieldVisitor; class AssertToSpaceInvariantObjectVisitor; class AssertToSpaceInvariantRefsVisitor; @@ -339,7 +353,7 @@ class ConcurrentCopying : public GarbageCollector { class DisableMarkingCheckpoint; class DisableWeakRefAccessCallback; class FlipCallback; - class GrayImmuneObjectVisitor; + template <bool kConcurrent> class GrayImmuneObjectVisitor; class ImmuneSpaceScanObjVisitor; class LostCopyVisitor; class RefFieldsVisitor; diff --git a/runtime/runtime.cc b/runtime/runtime.cc index eb068b3cad..b1acec6e82 100644 --- a/runtime/runtime.cc +++ b/runtime/runtime.cc @@ -336,6 +336,16 @@ Runtime::~Runtime() { jit_->DeleteThreadPool(); } + // Make sure our internal threads are dead before we start tearing down things they're using. + Dbg::StopJdwp(); + delete signal_catcher_; + + // Make sure all other non-daemon threads have terminated, and all daemon threads are suspended. + { + ScopedTrace trace2("Delete thread list"); + thread_list_->ShutDown(); + } + // TODO Maybe do some locking. for (auto& agent : agents_) { agent.Unload(); @@ -346,15 +356,9 @@ Runtime::~Runtime() { plugin.Unload(); } - // Make sure our internal threads are dead before we start tearing down things they're using. - Dbg::StopJdwp(); - delete signal_catcher_; + // Finally delete the thread list. + delete thread_list_; - // Make sure all other non-daemon threads have terminated, and all daemon threads are suspended. - { - ScopedTrace trace2("Delete thread list"); - delete thread_list_; - } // Delete the JIT after thread list to ensure that there is no remaining threads which could be // accessing the instrumentation when we delete it. if (jit_ != nullptr) { diff --git a/runtime/thread.cc b/runtime/thread.cc index 62a616b646..653a9bd1d4 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -129,12 +129,12 @@ static void UnimplementedEntryPoint() { } void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints); -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking); +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_active); void Thread::SetIsGcMarkingAndUpdateEntrypoints(bool is_marking) { CHECK(kUseReadBarrier); tls32_.is_gc_marking = is_marking; - UpdateReadBarrierEntrypoints(&tlsPtr_.quick_entrypoints, is_marking); + UpdateReadBarrierEntrypoints(&tlsPtr_.quick_entrypoints, /* is_active */ is_marking); ResetQuickAllocEntryPointsForThread(is_marking); } @@ -3604,4 +3604,9 @@ mirror::Object* Thread::GetPeerFromOtherThread() const { return peer; } +void Thread::SetReadBarrierEntrypoints() { + // Make sure entrypoints aren't null. + UpdateReadBarrierEntrypoints(&tlsPtr_.quick_entrypoints, /* is_active*/ true); +} + } // namespace art diff --git a/runtime/thread.h b/runtime/thread.h index 5251012cbb..6abde5b450 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -1180,6 +1180,9 @@ class Thread { return false; } + // Set to the read barrier marking entrypoints to be non-null. + void SetReadBarrierEntrypoints(); + static jobject CreateCompileTimePeer(JNIEnv* env, const char* name, bool as_daemon, diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc index 2e0d866c21..b63eaa40ef 100644 --- a/runtime/thread_list.cc +++ b/runtime/thread_list.cc @@ -73,12 +73,17 @@ ThreadList::ThreadList(uint64_t thread_suspend_timeout_ns) unregistering_count_(0), suspend_all_historam_("suspend all histogram", 16, 64), long_suspend_(false), + shut_down_(false), thread_suspend_timeout_ns_(thread_suspend_timeout_ns), empty_checkpoint_barrier_(new Barrier(0)) { CHECK(Monitor::IsValidLockWord(LockWord::FromThinLockId(kMaxThreadId, 1, 0U))); } ThreadList::~ThreadList() { + CHECK(shut_down_); +} + +void ThreadList::ShutDown() { ScopedTrace trace(__PRETTY_FUNCTION__); // Detach the current thread if necessary. If we failed to start, there might not be any threads. // We need to detach the current thread here in case there's another thread waiting to join with @@ -102,6 +107,8 @@ ThreadList::~ThreadList() { // TODO: there's an unaddressed race here where a thread may attach during shutdown, see // Thread::Init. SuspendAllDaemonThreadsForShutdown(); + + shut_down_ = true; } bool ThreadList::Contains(Thread* thread) { @@ -1362,6 +1369,7 @@ void ThreadList::SuspendAllDaemonThreadsForShutdown() { void ThreadList::Register(Thread* self) { DCHECK_EQ(self, Thread::Current()); + CHECK(!shut_down_); if (VLOG_IS_ON(threads)) { std::ostringstream oss; @@ -1387,13 +1395,14 @@ void ThreadList::Register(Thread* self) { CHECK(!Contains(self)); list_.push_back(self); if (kUseReadBarrier) { + gc::collector::ConcurrentCopying* const cc = + Runtime::Current()->GetHeap()->ConcurrentCopyingCollector(); // Initialize according to the state of the CC collector. - bool is_gc_marking = - Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsMarking(); - self->SetIsGcMarkingAndUpdateEntrypoints(is_gc_marking); - bool weak_ref_access_enabled = - Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsWeakRefAccessEnabled(); - self->SetWeakRefAccessEnabled(weak_ref_access_enabled); + self->SetIsGcMarkingAndUpdateEntrypoints(cc->IsMarking()); + if (cc->IsUsingReadBarrierEntrypoints()) { + self->SetReadBarrierEntrypoints(); + } + self->SetWeakRefAccessEnabled(cc->IsWeakRefAccessEnabled()); } } diff --git a/runtime/thread_list.h b/runtime/thread_list.h index 70917eb0f7..14bef5e2b9 100644 --- a/runtime/thread_list.h +++ b/runtime/thread_list.h @@ -50,6 +50,8 @@ class ThreadList { explicit ThreadList(uint64_t thread_suspend_timeout_ns); ~ThreadList(); + void ShutDown(); + void DumpForSigQuit(std::ostream& os) REQUIRES(!Locks::thread_list_lock_, !Locks::mutator_lock_); // For thread suspend timeout dumps. @@ -219,6 +221,10 @@ class ThreadList { // Whether or not the current thread suspension is long. bool long_suspend_; + // Whether the shutdown function has been called. This is checked in the destructor. It is an + // error to destroy a ThreadList instance without first calling ShutDown(). + bool shut_down_; + // Thread suspension timeout in nanoseconds. const uint64_t thread_suspend_timeout_ns_; diff --git a/test/623-checker-loop-regressions/src/Main.java b/test/623-checker-loop-regressions/src/Main.java index d1f36edb18..520e7c367c 100644 --- a/test/623-checker-loop-regressions/src/Main.java +++ b/test/623-checker-loop-regressions/src/Main.java @@ -341,6 +341,16 @@ public class Main { } } + // Bug b/37768917: potential dynamic BCE vs. loop optimizations + // case should be deal with correctly (used to DCHECK fail). + private static void arrayInTripCount(int[] a, byte[] b, int n) { + for (int k = 0; k < n; k++) { + for (int i = 0, u = a[0]; i < u; i++) { + b[i] += 2; + } + } + } + public static void main(String[] args) { expectEquals(10, earlyExitFirst(-1)); for (int i = 0; i <= 10; i++) { @@ -436,6 +446,13 @@ public class Main { expectEquals(dd[i], 1); } + xx[0] = 10; + byte[] bt = new byte[10]; + arrayInTripCount(xx, bt, 20); + for (int i = 0; i < bt.length; i++) { + expectEquals(40, bt[i]); + } + System.out.println("passed"); } diff --git a/test/648-inline-caches-unresolved/expected.txt b/test/648-inline-caches-unresolved/expected.txt new file mode 100644 index 0000000000..4e6a4384c5 --- /dev/null +++ b/test/648-inline-caches-unresolved/expected.txt @@ -0,0 +1 @@ +Subclass diff --git a/test/648-inline-caches-unresolved/info.txt b/test/648-inline-caches-unresolved/info.txt new file mode 100644 index 0000000000..8fc604281c --- /dev/null +++ b/test/648-inline-caches-unresolved/info.txt @@ -0,0 +1 @@ +Test for inlining with inline cache into an unresolved method. diff --git a/test/648-inline-caches-unresolved/profile b/test/648-inline-caches-unresolved/profile new file mode 100644 index 0000000000..92c0a41cab --- /dev/null +++ b/test/648-inline-caches-unresolved/profile @@ -0,0 +1 @@ +LMain;->inlineMonomorphicUnresolvedSuper(Ljava/lang/Object;)Ljava/lang/String;+LSubclass; diff --git a/test/648-inline-caches-unresolved/run b/test/648-inline-caches-unresolved/run new file mode 100644 index 0000000000..fb70d22867 --- /dev/null +++ b/test/648-inline-caches-unresolved/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec ${RUN} $@ --secondary --profile diff --git a/test/648-inline-caches-unresolved/src-dex2oat-unresolved/UnresolvedSuperClass.java b/test/648-inline-caches-unresolved/src-dex2oat-unresolved/UnresolvedSuperClass.java new file mode 100644 index 0000000000..dd3be00633 --- /dev/null +++ b/test/648-inline-caches-unresolved/src-dex2oat-unresolved/UnresolvedSuperClass.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class UnresolvedSuperClass { + public void superMethod() { + System.out.println("UnresolvedClass.superMethod()"); + } +} diff --git a/test/648-inline-caches-unresolved/src/Main.java b/test/648-inline-caches-unresolved/src/Main.java new file mode 100644 index 0000000000..4e8aeec171 --- /dev/null +++ b/test/648-inline-caches-unresolved/src/Main.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main extends UnresolvedSuperClass { + public static String inlineMonomorphicUnresolvedSuper(Object o) { + return o.toString(); + } + + public static void main(String[] args) { + System.out.println(inlineMonomorphicUnresolvedSuper(new Subclass())); + } +} + +class Subclass { + public String toString() { + return "Subclass"; + } +} diff --git a/test/912-classes/src/art/Test912Art.java b/test/912-classes/src/art/Test912Art.java index 6da3cadefe..a1e7ff2005 100644 --- a/test/912-classes/src/art/Test912Art.java +++ b/test/912-classes/src/art/Test912Art.java @@ -39,7 +39,7 @@ public class Test912Art { // run in configurations where dex2oat didn't verify the class itself. So explicitly // check whether the class has been already loaded, and skip then. // TODO: Add multiple configurations to the run script once that becomes easier to do. - if (hasJit() && !isLoadedClass("art.Test912Art$ClassD")) { + if (hasJit() && !isLoadedClass("Lart/Test912Art$ClassD;")) { testClassEventsJit(); } } diff --git a/test/etc/run-test-jar b/test/etc/run-test-jar index bb99e1cb50..f75055674e 100755 --- a/test/etc/run-test-jar +++ b/test/etc/run-test-jar @@ -564,6 +564,11 @@ if [ "$PROFILE" = "y" ] || [ "$RANDOM_PROFILE" = "y" ]; then profman_cmdline="${ANDROID_ROOT}/bin/profman \ --apk=$DEX_LOCATION/$TEST_NAME.jar \ --dex-location=$DEX_LOCATION/$TEST_NAME.jar" + if [ -f $DEX_LOCATION/$TEST_NAME-ex.jar ]; then + profman_cmdline="${profman_cmdline} \ + --apk=$DEX_LOCATION/$TEST_NAME-ex.jar \ + --dex-location=$DEX_LOCATION/$TEST_NAME-ex.jar" + fi COMPILE_FLAGS="${COMPILE_FLAGS} --profile-file=$DEX_LOCATION/$TEST_NAME.prof" FLAGS="${FLAGS} -Xcompiler-option --profile-file=$DEX_LOCATION/$TEST_NAME.prof" if [ "$PROFILE" = "y" ]; then diff --git a/test/knownfailures.json b/test/knownfailures.json index ea810db1ac..0a7089a765 100644 --- a/test/knownfailures.json +++ b/test/knownfailures.json @@ -328,11 +328,6 @@ "variant": "interpreter | optimizing | regalloc_gc | jit" }, { - "tests": ["912-classes"], - "bug": "http://b/36344364", - "variant": "no-dex2oat | relocate-npatchoat" - }, - { "tests": ["476-clinit-inline-static-invoke", "496-checker-inlining-class-loader", "508-referrer-method", |