diff options
139 files changed, 5905 insertions, 956 deletions
diff --git a/build/Android.bp b/build/Android.bp index 9156027dee..cd9d74a934 100644 --- a/build/Android.bp +++ b/build/Android.bp @@ -145,6 +145,10 @@ art_global_defaults { "external/vixl/src", "external/zlib", ], + + tidy_checks: [ + "-google-default-arguments", + ], } art_debug_defaults { diff --git a/cmdline/cmdline_types.h b/cmdline/cmdline_types.h index 13a323533c..a5bb117509 100644 --- a/cmdline/cmdline_types.h +++ b/cmdline/cmdline_types.h @@ -182,7 +182,7 @@ template <size_t Divisor> struct CmdlineType<Memory<Divisor>> : CmdlineTypeParser<Memory<Divisor>> { using typename CmdlineTypeParser<Memory<Divisor>>::Result; - Result Parse(const std::string arg) { + Result Parse(const std::string& arg) { CMDLINE_DEBUG_LOG << "Parsing memory: " << arg << std::endl; size_t val = ParseMemoryOption(arg.c_str(), Divisor); CMDLINE_DEBUG_LOG << "Memory parsed to size_t value: " << val << std::endl; diff --git a/compiler/Android.bp b/compiler/Android.bp index 0b148598cf..f1bf27ece7 100644 --- a/compiler/Android.bp +++ b/compiler/Android.bp @@ -105,6 +105,7 @@ art_cc_defaults { "optimizing/instruction_simplifier_arm.cc", "optimizing/instruction_simplifier_shared.cc", "optimizing/intrinsics_arm.cc", + "optimizing/intrinsics_arm_vixl.cc", "utils/arm/assembler_arm.cc", "utils/arm/assembler_arm_vixl.cc", "utils/arm/assembler_thumb2.cc", @@ -203,7 +204,8 @@ art_cc_defaults { gensrcs { name: "art_compiler_operator_srcs", - cmd: "art/tools/generate-operator-out.py art/compiler $in > $out", + cmd: "$(location generate-operator-out.py) art/compiler $(in) > $(out)", + tool_files: ["generate-operator-out.py"], srcs: [ "compiled_method.h", "dex/dex_to_dex_compiler.h", diff --git a/compiler/dex/quick_compiler_callbacks.h b/compiler/dex/quick_compiler_callbacks.h index 34fd88b498..db0fdaa72f 100644 --- a/compiler/dex/quick_compiler_callbacks.h +++ b/compiler/dex/quick_compiler_callbacks.h @@ -18,6 +18,7 @@ #define ART_COMPILER_DEX_QUICK_COMPILER_CALLBACKS_H_ #include "compiler_callbacks.h" +#include "verifier/verifier_deps.h" namespace art { @@ -46,16 +47,16 @@ class QuickCompilerCallbacks FINAL : public CompilerCallbacks { } verifier::VerifierDeps* GetVerifierDeps() const OVERRIDE { - return verifier_deps_; + return verifier_deps_.get(); } - void SetVerifierDeps(verifier::VerifierDeps* deps) { - verifier_deps_ = deps; + void SetVerifierDeps(verifier::VerifierDeps* deps) OVERRIDE { + verifier_deps_.reset(deps); } private: VerificationResults* const verification_results_; - verifier::VerifierDeps* verifier_deps_; + std::unique_ptr<verifier::VerifierDeps> verifier_deps_; }; } // namespace art diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 1b87725230..223be884b0 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -39,6 +39,7 @@ #include "compiled_class.h" #include "compiled_method.h" #include "compiler.h" +#include "compiler_callbacks.h" #include "compiler_driver-inl.h" #include "dex_compilation_unit.h" #include "dex_file-inl.h" @@ -393,6 +394,7 @@ static void SetupIntrinsic(Thread* self, void CompilerDriver::CompileAll(jobject class_loader, const std::vector<const DexFile*>& dex_files, + verifier::VerifierDeps* verifier_deps, TimingLogger* timings) { DCHECK(!Runtime::Current()->IsStarted()); @@ -404,7 +406,7 @@ void CompilerDriver::CompileAll(jobject class_loader, // 2) Resolve all classes // 3) Attempt to verify all classes // 4) Attempt to initialize image classes, and trivially initialized classes - PreCompile(class_loader, dex_files, timings); + PreCompile(class_loader, dex_files, verifier_deps, timings); if (GetCompilerOptions().IsBootImage()) { // We don't need to setup the intrinsics for non boot image compilation, as // those compilations will pick up a boot image that have the ArtMethod already @@ -676,7 +678,7 @@ void CompilerDriver::CompileOne(Thread* self, ArtMethod* method, TimingLogger* t InitializeThreadPools(); - PreCompile(jclass_loader, dex_files, timings); + PreCompile(jclass_loader, dex_files, /* verifier_deps */ nullptr, timings); // Can we run DEX-to-DEX compiler on this class ? optimizer::DexToDexCompilationLevel dex_to_dex_compilation_level = @@ -873,6 +875,7 @@ inline void CompilerDriver::CheckThreadPools() { void CompilerDriver::PreCompile(jobject class_loader, const std::vector<const DexFile*>& dex_files, + verifier::VerifierDeps* verifier_deps, TimingLogger* timings) { CheckThreadPools(); @@ -906,7 +909,7 @@ void CompilerDriver::PreCompile(jobject class_loader, VLOG(compiler) << "Resolve const-strings: " << GetMemoryUsageString(false); } - Verify(class_loader, dex_files, timings); + Verify(class_loader, dex_files, verifier_deps, timings); VLOG(compiler) << "Verify: " << GetMemoryUsageString(false); if (had_hard_verifier_failure_ && GetCompilerOptions().AbortOnHardVerifierFailure()) { @@ -1522,7 +1525,7 @@ void CompilerDriver::GetCodeAndMethodForDirectCall(const mirror::Class* referrer if (!use_dex_cache) { bool method_in_image = false; - const std::vector<gc::space::ImageSpace*> image_spaces = heap->GetBootImageSpaces(); + const std::vector<gc::space::ImageSpace*>& image_spaces = heap->GetBootImageSpaces(); for (gc::space::ImageSpace* image_space : image_spaces) { const auto& method_section = image_space->GetImageHeader().GetMethodsSection(); if (method_section.Contains(reinterpret_cast<uint8_t*>(method) - image_space->Begin())) { @@ -1932,15 +1935,61 @@ void CompilerDriver::SetVerified(jobject class_loader, } } -void CompilerDriver::Verify(jobject class_loader, +void CompilerDriver::Verify(jobject jclass_loader, const std::vector<const DexFile*>& dex_files, + verifier::VerifierDeps* verifier_deps, TimingLogger* timings) { + if (verifier_deps != nullptr) { + TimingLogger::ScopedTiming t("Fast Verify", timings); + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<2> hs(soa.Self()); + Handle<mirror::ClassLoader> class_loader( + hs.NewHandle(soa.Decode<mirror::ClassLoader>(jclass_loader))); + MutableHandle<mirror::Class> cls(hs.NewHandle<mirror::Class>(nullptr)); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + if (verifier_deps->ValidateDependencies(class_loader, soa.Self())) { + // We successfully validated the dependencies, now update class status + // of verified classes. Note that the dependencies also record which classes + // could not be fully verified; we could try again, but that would hurt verification + // time. So instead we assume these classes still need to be verified at + // runtime. + for (const DexFile* dex_file : dex_files) { + // Fetch the list of unverified classes and turn it into a set for faster + // lookups. + const std::vector<uint16_t>& unverified_classes = + verifier_deps->GetUnverifiedClasses(*dex_file); + std::set<uint16_t> set(unverified_classes.begin(), unverified_classes.end()); + for (uint32_t i = 0; i < dex_file->NumClassDefs(); ++i) { + const DexFile::ClassDef& class_def = dex_file->GetClassDef(i); + const char* descriptor = dex_file->GetClassDescriptor(class_def); + cls.Assign(class_linker->FindClass(soa.Self(), descriptor, class_loader)); + if (cls.Get() == nullptr) { + CHECK(soa.Self()->IsExceptionPending()); + soa.Self()->ClearException(); + } else if (set.find(class_def.class_idx_) == set.end()) { + ObjectLock<mirror::Class> lock(soa.Self(), cls); + mirror::Class::SetStatus(cls, mirror::Class::kStatusVerified, soa.Self()); + } + } + } + return; + } + } + + // If there is no passed `verifier_deps` (because of non-existing vdex), or + // the passed `verifier_deps` is not valid anymore, create a new one for + // non boot image compilation. The verifier will need it to record the new dependencies. + // Then dex2oat can update the vdex file with these new dependencies. + if (!GetCompilerOptions().IsBootImage()) { + Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps( + new verifier::VerifierDeps(dex_files)); + } // Note: verification should not be pulling in classes anymore when compiling the boot image, // as all should have been resolved before. As such, doing this in parallel should still // be deterministic. for (const DexFile* dex_file : dex_files) { CHECK(dex_file != nullptr); - VerifyDexFile(class_loader, + VerifyDexFile(jclass_loader, *dex_file, dex_files, parallel_thread_pool_.get(), diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h index 4a48f9c841..c8d6cb0d4d 100644 --- a/compiler/driver/compiler_driver.h +++ b/compiler/driver/compiler_driver.h @@ -50,6 +50,7 @@ class DexCache; namespace verifier { class MethodVerifier; +class VerifierDeps; class VerifierDepsTest; } // namespace verifier @@ -117,6 +118,7 @@ class CompilerDriver { void CompileAll(jobject class_loader, const std::vector<const DexFile*>& dex_files, + verifier::VerifierDeps* verifier_deps, TimingLogger* timings) REQUIRES(!Locks::mutator_lock_, !compiled_classes_lock_, !dex_to_dex_references_lock_); @@ -415,6 +417,7 @@ class CompilerDriver { private: void PreCompile(jobject class_loader, const std::vector<const DexFile*>& dex_files, + verifier::VerifierDeps* verifier_deps, TimingLogger* timings) REQUIRES(!Locks::mutator_lock_, !compiled_classes_lock_); @@ -437,7 +440,9 @@ class CompilerDriver { void Verify(jobject class_loader, const std::vector<const DexFile*>& dex_files, + verifier::VerifierDeps* verifier_deps, TimingLogger* timings); + void VerifyDexFile(jobject class_loader, const DexFile& dex_file, const std::vector<const DexFile*>& dex_files, diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc index 845028d427..9679a796cb 100644 --- a/compiler/driver/compiler_driver_test.cc +++ b/compiler/driver/compiler_driver_test.cc @@ -43,6 +43,7 @@ class CompilerDriverTest : public CommonCompilerTest { TimingLogger::ScopedTiming t(__FUNCTION__, &timings); compiler_driver_->CompileAll(class_loader, GetDexFiles(class_loader), + /* verifier_deps */ nullptr, &timings); t.NewTiming("MakeAllExecutable"); MakeAllExecutable(class_loader); diff --git a/compiler/generate-operator-out.py b/compiler/generate-operator-out.py new file mode 120000 index 0000000000..cc291d20c1 --- /dev/null +++ b/compiler/generate-operator-out.py @@ -0,0 +1 @@ +../tools/generate-operator-out.py
\ No newline at end of file diff --git a/compiler/image_test.cc b/compiler/image_test.cc index 8fdf6fca68..fcb8979459 100644 --- a/compiler/image_test.cc +++ b/compiler/image_test.cc @@ -189,7 +189,7 @@ void CompilationHelper::Compile(CompilerDriver* driver, TimingLogger timings("ImageTest::WriteRead", false, false); TimingLogger::ScopedTiming t("CompileAll", &timings); driver->SetDexFilesForOatFile(class_path); - driver->CompileAll(class_loader, class_path, &timings); + driver->CompileAll(class_loader, class_path, /* verifier_deps */ nullptr, &timings); t.NewTiming("WriteElf"); SafeMap<std::string, std::string> key_value_store; diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index a37bf4b9fe..51ef440c7f 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -2070,13 +2070,8 @@ void ImageWriter::FixupClass(mirror::Class* orig, mirror::Class* copy) { void ImageWriter::FixupObject(Object* orig, Object* copy) { DCHECK(orig != nullptr); DCHECK(copy != nullptr); - if (kUseBakerOrBrooksReadBarrier) { - orig->AssertReadBarrierPointer(); - if (kUseBrooksReadBarrier) { - // Note the address 'copy' isn't the same as the image address of 'orig'. - copy->SetReadBarrierPointer(GetImageAddress(orig)); - DCHECK_EQ(copy->GetReadBarrierPointer(), GetImageAddress(orig)); - } + if (kUseBakerReadBarrier) { + orig->AssertReadBarrierState(); } auto* klass = orig->GetClass(); if (klass->IsIntArrayClass() || klass->IsLongArrayClass()) { diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index 64ee574889..102637f01b 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -377,7 +377,8 @@ TEST_F(OatTest, WriteRead) { if (kCompile) { TimingLogger timings2("OatTest::WriteRead", false, false); compiler_driver_->SetDexFilesForOatFile(class_linker->GetBootClassPath()); - compiler_driver_->CompileAll(class_loader, class_linker->GetBootClassPath(), &timings2); + compiler_driver_->CompileAll( + class_loader, class_linker->GetBootClassPath(), /* verifier_deps */ nullptr, &timings2); } ScratchFile tmp_oat, tmp_vdex(tmp_oat, ".vdex"); @@ -391,7 +392,8 @@ TEST_F(OatTest, WriteRead) { ASSERT_TRUE(success); if (kCompile) { // OatWriter strips the code, regenerate to compare - compiler_driver_->CompileAll(class_loader, class_linker->GetBootClassPath(), &timings); + compiler_driver_->CompileAll( + class_loader, class_linker->GetBootClassPath(), /* verifier_deps */ nullptr, &timings); } std::unique_ptr<OatFile> oat_file(OatFile::Open(tmp_oat.GetFilename(), tmp_oat.GetFilename(), @@ -515,7 +517,7 @@ TEST_F(OatTest, EmptyTextSection) { soa.Decode<mirror::ClassLoader>(class_loader).Ptr()); } compiler_driver_->SetDexFilesForOatFile(dex_files); - compiler_driver_->CompileAll(class_loader, dex_files, &timings); + compiler_driver_->CompileAll(class_loader, dex_files, /* verifier_deps */ nullptr, &timings); ScratchFile tmp_oat, tmp_vdex(tmp_oat, ".vdex"); SafeMap<std::string, std::string> key_value_store; diff --git a/compiler/optimizing/bounds_check_elimination.cc b/compiler/optimizing/bounds_check_elimination.cc index d2357a5d05..7dc094b25f 100644 --- a/compiler/optimizing/bounds_check_elimination.cc +++ b/compiler/optimizing/bounds_check_elimination.cc @@ -548,7 +548,21 @@ class BCEVisitor : public HGraphVisitor { void VisitBasicBlock(HBasicBlock* block) OVERRIDE { DCHECK(!IsAddedBlock(block)); first_index_bounds_check_map_.clear(); - HGraphVisitor::VisitBasicBlock(block); + // Visit phis and instructions using a safe iterator. The iteration protects + // against deleting the current instruction during iteration. However, it + // must advance next_ if that instruction is deleted during iteration. + for (HInstruction* instruction = block->GetFirstPhi(); instruction != nullptr;) { + DCHECK(instruction->IsInBlock()); + next_ = instruction->GetNext(); + instruction->Accept(this); + instruction = next_; + } + for (HInstruction* instruction = block->GetFirstInstruction(); instruction != nullptr;) { + DCHECK(instruction->IsInBlock()); + next_ = instruction->GetNext(); + instruction->Accept(this); + instruction = next_; + } // We should never deoptimize from an osr method, otherwise we might wrongly optimize // code dominated by the deoptimization. if (!GetGraph()->IsCompilingOsr()) { @@ -1798,7 +1812,12 @@ class BCEVisitor : public HGraphVisitor { } /** Helper method to replace an instruction with another instruction. */ - static void ReplaceInstruction(HInstruction* instruction, HInstruction* replacement) { + void ReplaceInstruction(HInstruction* instruction, HInstruction* replacement) { + // Safe iteration. + if (instruction == next_) { + next_ = next_->GetNext(); + } + // Replace and remove. instruction->ReplaceWith(replacement); instruction->GetBlock()->RemoveInstruction(instruction); } @@ -1831,6 +1850,9 @@ class BCEVisitor : public HGraphVisitor { // Range analysis based on induction variables. InductionVarRange induction_range_; + // Safe iteration. + HInstruction* next_; + DISALLOW_COPY_AND_ASSIGN(BCEVisitor); }; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index be65f89ef1..7cab97d2e5 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -638,6 +638,11 @@ class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) << "Unexpected instruction in read barrier marking slow path: " << instruction_->DebugName(); + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. + DCHECK(!(instruction_->IsArrayGet() && + instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); __ Bind(GetEntryLabel()); // No need to save live registers; it's taken care of by the @@ -894,6 +899,11 @@ class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) << "Unexpected instruction in read barrier for heap reference slow path: " << instruction_->DebugName(); + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. + DCHECK(!(instruction_->IsArrayGet() && + instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); @@ -4841,8 +4851,6 @@ void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { instruction->IsStringCharAt(); HInstruction* array_instr = instruction->GetArray(); bool has_intermediate_address = array_instr->IsIntermediateAddress(); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); switch (type) { case Primitive::kPrimBoolean: @@ -4915,6 +4923,11 @@ void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimNot: { + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. + DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); + static_assert( sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); @@ -5055,8 +5068,6 @@ void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { Location value_loc = locations->InAt(2); HInstruction* array_instr = instruction->GetArray(); bool has_intermediate_address = array_instr->IsIntermediateAddress(); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); switch (value_type) { case Primitive::kPrimBoolean: @@ -5306,8 +5317,6 @@ void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { } void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); @@ -5322,9 +5331,6 @@ void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* Location first = locations->InAt(0); Location second = locations->InAt(1); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); - if (second.IsRegister()) { __ add(out.AsRegister<Register>(), first.AsRegister<Register>(), @@ -6843,7 +6849,7 @@ void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* i // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. // } @@ -6919,14 +6925,13 @@ void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* i } AddSlowPath(slow_path); - // if (rb_state == ReadBarrier::gray_ptr_) + // if (rb_state == ReadBarrier::GrayState()) // ref = ReadBarrier::Mark(ref); // Given the numeric representation, it's enough to check the low bit of the // rb_state. We do that by shifting the bit out of the lock word with LSRS // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. __ Bind(slow_path->GetExitLabel()); diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index b53750966d..d868984387 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -626,6 +626,11 @@ class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 { (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) << "Unexpected instruction in read barrier marking slow path: " << instruction_->DebugName(); + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. + DCHECK(!(instruction_->IsArrayGet() && + instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); __ Bind(GetEntryLabel()); // No need to save live registers; it's taken care of by the @@ -876,7 +881,9 @@ class ReadBarrierForHeapReferenceSlowPathARM64 : public SlowPathCodeARM64 { (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) << "Unexpected instruction in read barrier for heap reference slow path: " << instruction_->DebugName(); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. + // The read barrier instrumentation of object ArrayGet + // instructions does not support the HIntermediateAddress + // instruction. DCHECK(!(instruction_->IsArrayGet() && instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); @@ -2192,8 +2199,6 @@ void InstructionCodeGeneratorARM64::VisitArm64DataProcWithShifterOp( } void LocationsBuilderARM64::VisitIntermediateAddress(HIntermediateAddress* instruction) { - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); @@ -2201,10 +2206,7 @@ void LocationsBuilderARM64::VisitIntermediateAddress(HIntermediateAddress* instr locations->SetOut(Location::RequiresRegister()); } -void InstructionCodeGeneratorARM64::VisitIntermediateAddress( - HIntermediateAddress* instruction) { - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); +void InstructionCodeGeneratorARM64::VisitIntermediateAddress(HIntermediateAddress* instruction) { __ Add(OutputRegister(instruction), InputRegisterAt(instruction, 0), Operand(InputOperandAt(instruction, 1))); @@ -2304,11 +2306,15 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { // Block pools between `Load` and `MaybeRecordImplicitNullCheck`. BlockPoolsScope block_pools(masm); + // The read barrier instrumentation of object ArrayGet instructions + // does not support the HIntermediateAddress instruction. + DCHECK(!((type == Primitive::kPrimNot) && + instruction->GetArray()->IsIntermediateAddress() && + kEmitCompilerReadBarrier)); + if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { // Object ArrayGet with Baker's read barrier case. Register temp = temps.AcquireW(); - // The read barrier instrumentation does not support the HIntermediateAddress instruction yet. - DCHECK(!instruction->GetArray()->IsIntermediateAddress()); // Note that a potential implicit null check is handled in the // CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier call. codegen_->GenerateArrayLoadWithBakerReadBarrier( @@ -2341,9 +2347,6 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { } else { Register temp = temps.AcquireSameSizeAs(obj); if (instruction->GetArray()->IsIntermediateAddress()) { - // The read barrier instrumentation does not support the - // HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); // We do not need to compute the intermediate address from the array: the // input instruction has done it already. See the comment in // `TryExtractArrayAccessAddress()`. @@ -2451,9 +2454,6 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { UseScratchRegisterScope temps(masm); Register temp = temps.AcquireSameSizeAs(array); if (instruction->GetArray()->IsIntermediateAddress()) { - // The read barrier instrumentation does not support the - // HIntermediateAddress instruction yet. - DCHECK(!kEmitCompilerReadBarrier); // We do not need to compute the intermediate address from the array: the // input instruction has done it already. See the comment in // `TryExtractArrayAccessAddress()`. @@ -5426,7 +5426,7 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. // } @@ -5517,12 +5517,11 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* } AddSlowPath(slow_path); - // if (rb_state == ReadBarrier::gray_ptr_) + // if (rb_state == ReadBarrier::GrayState()) // ref = ReadBarrier::Mark(ref); // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); __ Tbnz(temp, LockWord::kReadBarrierStateShift, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index c00ee553ef..e69528e43f 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -23,6 +23,7 @@ #include "compiled_method.h" #include "entrypoints/quick/quick_entrypoints.h" #include "gc/accounting/card_table.h" +#include "intrinsics_arm_vixl.h" #include "mirror/array-inl.h" #include "mirror/class-inl.h" #include "thread.h" @@ -1474,19 +1475,36 @@ void LocationsBuilderARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* i // art::PrepareForRegisterAllocation. DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); - // TODO(VIXL): TryDispatch + IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_); + if (intrinsic.TryDispatch(invoke)) { + if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { + invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); + } + return; + } HandleInvoke(invoke); // TODO(VIXL): invoke->HasPcRelativeDexCache() } +static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARMVIXL* codegen) { + if (invoke->GetLocations()->Intrinsified()) { + IntrinsicCodeGeneratorARMVIXL intrinsic(codegen); + intrinsic.Dispatch(invoke); + return true; + } + return false; +} + void InstructionCodeGeneratorARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { // Explicit clinit checks triggered by static invokes must have been pruned by // art::PrepareForRegisterAllocation. DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); - // TODO(VIXL): TryGenerateIntrinsicCode + if (TryGenerateIntrinsicCode(invoke, codegen_)) { + return; + } LocationSummary* locations = invoke->GetLocations(); DCHECK(locations->HasTemps()); @@ -1502,13 +1520,18 @@ void LocationsBuilderARMVIXL::HandleInvoke(HInvoke* invoke) { } void LocationsBuilderARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) { - // TODO(VIXL): TryDispatch + IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_); + if (intrinsic.TryDispatch(invoke)) { + return; + } HandleInvoke(invoke); } void InstructionCodeGeneratorARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) { - // TODO(VIXL): TryGenerateIntrinsicCode + if (TryGenerateIntrinsicCode(invoke, codegen_)) { + return; + } codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); @@ -5039,6 +5062,39 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( } } +void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier( + HInstruction* instruction ATTRIBUTE_UNUSED, + Location ref ATTRIBUTE_UNUSED, + vixl::aarch32::Register obj ATTRIBUTE_UNUSED, + uint32_t offset ATTRIBUTE_UNUSED, + Location temp ATTRIBUTE_UNUSED, + bool needs_null_check ATTRIBUTE_UNUSED) { + TODO_VIXL32(FATAL); +} + +void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier( + HInstruction* instruction ATTRIBUTE_UNUSED, + Location ref ATTRIBUTE_UNUSED, + vixl::aarch32::Register obj ATTRIBUTE_UNUSED, + uint32_t offset ATTRIBUTE_UNUSED, + Location index ATTRIBUTE_UNUSED, + ScaleFactor scale_factor ATTRIBUTE_UNUSED, + Location temp ATTRIBUTE_UNUSED, + bool needs_null_check ATTRIBUTE_UNUSED, + bool always_update_field ATTRIBUTE_UNUSED, + vixl::aarch32::Register* temp2 ATTRIBUTE_UNUSED) { + TODO_VIXL32(FATAL); +} + +void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction ATTRIBUTE_UNUSED, + Location out ATTRIBUTE_UNUSED, + Location ref ATTRIBUTE_UNUSED, + Location obj ATTRIBUTE_UNUSED, + uint32_t offset ATTRIBUTE_UNUSED, + Location index ATTRIBUTE_UNUSED) { + TODO_VIXL32(FATAL); +} + void CodeGeneratorARMVIXL::MaybeGenerateReadBarrierSlow(HInstruction* instruction ATTRIBUTE_UNUSED, Location out, Location ref ATTRIBUTE_UNUSED, diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index efd33c7025..2f946e4263 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -7093,7 +7093,7 @@ void CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* i // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. // } @@ -7111,14 +7111,13 @@ void CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* i uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte; constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte; constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position); - // if (rb_state == ReadBarrier::gray_ptr_) + // if (rb_state == ReadBarrier::GrayState()) // ref = ReadBarrier::Mark(ref); // At this point, just do the "if" and make sure that flags are preserved until the branch. __ testb(Address(obj, monitor_offset + gray_byte_position), Immediate(test_value)); diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index fcabeeae5d..232c3b3cbb 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -5740,7 +5740,19 @@ void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) { CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); } -static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { +static bool CheckCastTypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { + if (type_check_kind == TypeCheckKind::kInterfaceCheck && !kPoisonHeapReferences) { + // We need a temporary for holding the iftable length. + return true; + } + return kEmitCompilerReadBarrier && + !kUseBakerReadBarrier && + (type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck); +} + +static bool InstanceOfTypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { return kEmitCompilerReadBarrier && !kUseBakerReadBarrier && (type_check_kind == TypeCheckKind::kAbstractClassCheck || @@ -5778,7 +5790,7 @@ void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) { locations->SetOut(Location::RequiresRegister()); // When read barriers are enabled, we need a temporary register for // some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { + if (InstanceOfTypeCheckNeedsATemporary(type_check_kind)) { locations->AddTemp(Location::RequiresRegister()); } } @@ -5791,7 +5803,7 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { Location cls = locations->InAt(1); Location out_loc = locations->Out(); CpuRegister out = out_loc.AsRegister<CpuRegister>(); - Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? + Location maybe_temp_loc = InstanceOfTypeCheckNeedsATemporary(type_check_kind) ? locations->GetTemp(0) : Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); @@ -5809,7 +5821,11 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { } // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); switch (type_check_kind) { case TypeCheckKind::kExactCheck: { @@ -5970,33 +5986,45 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { } } -void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) { - LocationSummary::CallKind call_kind = LocationSummary::kNoCall; - bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); +bool IsTypeCheckSlowPathFatal(TypeCheckKind type_check_kind, bool throws_into_catch) { switch (type_check_kind) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kAbstractClassCheck: case TypeCheckKind::kClassHierarchyCheck: case TypeCheckKind::kArrayObjectCheck: - call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. - break; + return !throws_into_catch && !kEmitCompilerReadBarrier; + case TypeCheckKind::kInterfaceCheck: + return !throws_into_catch && !kEmitCompilerReadBarrier && !kPoisonHeapReferences; case TypeCheckKind::kArrayCheck: case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - call_kind = LocationSummary::kCallOnSlowPath; - break; + return false; } + LOG(FATAL) << "Unreachable"; + UNREACHABLE(); +} + +void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) { + bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); + bool is_fatal_slow_path = IsTypeCheckSlowPathFatal(type_check_kind, throws_into_catch); + LocationSummary::CallKind call_kind = is_fatal_slow_path + ? LocationSummary::kNoCall + : LocationSummary::kCallOnSlowPath; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::Any()); + if (type_check_kind == TypeCheckKind::kInterfaceCheck) { + // Require a register for the interface check since there is a loop that compares the class to + // a memory address. + locations->SetInAt(1, Location::RequiresRegister()); + } else { + locations->SetInAt(1, Location::Any()); + } + // Note that TypeCheckSlowPathX86_64 uses this "temp" register too. locations->AddTemp(Location::RequiresRegister()); // When read barriers are enabled, we need an additional temporary // register for some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { + if (CheckCastTypeCheckNeedsATemporary(type_check_kind)) { locations->AddTemp(Location::RequiresRegister()); } } @@ -6009,20 +6037,19 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { Location cls = locations->InAt(1); Location temp_loc = locations->GetTemp(0); CpuRegister temp = temp_loc.AsRegister<CpuRegister>(); - Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? + Location maybe_temp2_loc = CheckCastTypeCheckNeedsATemporary(type_check_kind) ? locations->GetTemp(1) : Location::NoLocation(); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); + const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); + const int object_array_data_offset = mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); bool is_type_check_slow_path_fatal = - (type_check_kind == TypeCheckKind::kExactCheck || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck) && - !instruction->CanThrowIntoCatchBlock(); + IsTypeCheckSlowPathFatal(type_check_kind, instruction->CanThrowIntoCatchBlock()); SlowPathCode* type_check_slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(instruction, is_type_check_slow_path_fatal); @@ -6039,8 +6066,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { } // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); - + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); if (cls.IsRegister()) { __ cmpl(temp, cls.AsRegister<CpuRegister>()); } else { @@ -6063,8 +6093,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { } // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); - + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. NearLabel loop, compare_classes; @@ -6083,7 +6116,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&compare_classes); @@ -6107,8 +6144,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { } // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); - + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); // Walk over the class hierarchy to find a match. NearLabel loop; __ Bind(&loop); @@ -6133,7 +6173,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&done); break; @@ -6152,8 +6196,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { } // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); - + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); // Do an exact check. NearLabel check_non_primitive_component_type; if (cls.IsRegister()) { @@ -6180,7 +6227,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&check_non_primitive_component_type); @@ -6188,7 +6239,11 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { __ j(kEqual, &done); // Same comment as above regarding `temp` and the slow path. // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&done); break; @@ -6197,17 +6252,15 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kUnresolvedCheck: case TypeCheckKind::kInterfaceCheck: NearLabel done; + // Avoid null check if we know obj is not null. if (instruction->MustDoNullCheck()) { __ testl(obj, obj); __ j(kEqual, &done); } - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); - // We always go into the type check slow path for the unresolved - // and interface check cases. + // We always go into the type check slow path for the unresolved case. // // We cannot directly call the CheckCast runtime entry point // without resorting to a type checking slow path here (i.e. by @@ -6223,6 +6276,53 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // case of the `switch` code as it was previously (with a direct // call to the runtime not using a type checking slow path). // This should also be beneficial for the other cases above. + + // Fast path for the interface check. Since we compare with a memory location in the inner + // loop we would need to have cls poisoned. However unpoisoning cls would reset the + // conditional flags and cause the conditional jump to be incorrect. + if (type_check_kind == TypeCheckKind::kInterfaceCheck && !kPoisonHeapReferences) { + // Try to avoid read barriers to improve the fast path. We can not get false positives by + // doing this. + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + /*emit_read_barrier*/ false); + + // /* HeapReference<Class> */ temp = temp->iftable_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + temp_loc, + iftable_offset, + /*emit_read_barrier*/ false); + NearLabel is_null; + // Null iftable means it is empty. + __ testl(temp_loc.AsRegister<CpuRegister>(), temp_loc.AsRegister<CpuRegister>()); + __ j(kZero, &is_null); + + // Loop through the iftable and check if any class matches. + __ movl(maybe_temp2_loc.AsRegister<CpuRegister>(), + Address(temp_loc.AsRegister<CpuRegister>(), array_length_offset)); + + NearLabel start_loop; + __ Bind(&start_loop); + __ cmpl(cls.AsRegister<CpuRegister>(), + Address(temp_loc.AsRegister<CpuRegister>(), object_array_data_offset)); + __ j(kEqual, &done); // Return if same class. + // Go to next interface. + __ addq(temp_loc.AsRegister<CpuRegister>(), Immediate(2 * kHeapReferenceSize)); + __ subq(maybe_temp2_loc.AsRegister<CpuRegister>(), Immediate(2)); + __ j(kNotZero, &start_loop); + __ Bind(&is_null); + } + + // Since we clobbered temp_loc holding the class, we need to reload it. + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&done); break; @@ -6397,10 +6497,11 @@ void InstructionCodeGeneratorX86_64::GenerateReferenceLoadOneRegister(HInstructi void InstructionCodeGeneratorX86_64::GenerateReferenceLoadTwoRegisters(HInstruction* instruction, Location out, Location obj, - uint32_t offset) { + uint32_t offset, + bool emit_read_barrier) { CpuRegister out_reg = out.AsRegister<CpuRegister>(); CpuRegister obj_reg = obj.AsRegister<CpuRegister>(); - if (kEmitCompilerReadBarrier) { + if (emit_read_barrier) { if (kUseBakerReadBarrier) { // Load with fast path based Baker's read barrier. // /* HeapReference<Object> */ out = *(obj + offset) @@ -6535,7 +6636,7 @@ void CodeGeneratorX86_64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. // } @@ -6553,14 +6654,13 @@ void CodeGeneratorX86_64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte; constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte; constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position); - // if (rb_state == ReadBarrier::gray_ptr_) + // if (rb_state == ReadBarrier::GrayState()) // ref = ReadBarrier::Mark(ref); // At this point, just do the "if" and make sure that flags are preserved until the branch. __ testb(Address(obj, monitor_offset + gray_byte_position), Immediate(test_value)); diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 8b19dad0d0..5a6dc54e7a 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -248,7 +248,8 @@ class InstructionCodeGeneratorX86_64 : public InstructionCodeGenerator { void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, Location out, Location obj, - uint32_t offset); + uint32_t offset, + bool emit_read_barrier); // Generate a GC root reference load: // // root <- *address diff --git a/compiler/optimizing/instruction_simplifier_arm.h b/compiler/optimizing/instruction_simplifier_arm.h index 782110c40a..9b54511340 100644 --- a/compiler/optimizing/instruction_simplifier_arm.h +++ b/compiler/optimizing/instruction_simplifier_arm.h @@ -48,7 +48,7 @@ class InstructionSimplifierArmVisitor : public HGraphVisitor { class InstructionSimplifierArm : public HOptimization { public: InstructionSimplifierArm(HGraph* graph, OptimizingCompilerStats* stats) - : HOptimization(graph, kInstructionSimplifierArmPassName, stats) {} + : HOptimization(graph, kInstructionSimplifierArmPassName, stats) {} static constexpr const char* kInstructionSimplifierArmPassName = "instruction_simplifier_arm"; diff --git a/compiler/optimizing/instruction_simplifier_arm64.h b/compiler/optimizing/instruction_simplifier_arm64.h index f71684efe9..d4cb1f14b7 100644 --- a/compiler/optimizing/instruction_simplifier_arm64.h +++ b/compiler/optimizing/instruction_simplifier_arm64.h @@ -82,9 +82,10 @@ class InstructionSimplifierArm64Visitor : public HGraphVisitor { class InstructionSimplifierArm64 : public HOptimization { public: InstructionSimplifierArm64(HGraph* graph, OptimizingCompilerStats* stats) - : HOptimization(graph, kInstructionSimplifierArm64PassName, stats) {} - static constexpr const char* kInstructionSimplifierArm64PassName - = "instruction_simplifier_arm64"; + : HOptimization(graph, kInstructionSimplifierArm64PassName, stats) {} + + static constexpr const char* kInstructionSimplifierArm64PassName = "instruction_simplifier_arm64"; + void Run() OVERRIDE { InstructionSimplifierArm64Visitor visitor(graph_, stats_); visitor.VisitReversePostOrder(); diff --git a/compiler/optimizing/instruction_simplifier_shared.cc b/compiler/optimizing/instruction_simplifier_shared.cc index 04e063c92e..c2b1374f62 100644 --- a/compiler/optimizing/instruction_simplifier_shared.cc +++ b/compiler/optimizing/instruction_simplifier_shared.cc @@ -231,15 +231,6 @@ bool TryExtractArrayAccessAddress(HInstruction* access, HInstruction* array, HInstruction* index, size_t data_offset) { - if (kEmitCompilerReadBarrier) { - // The read barrier instrumentation does not support the - // HIntermediateAddress instruction yet. - // - // TODO: Handle this case properly in the ARM64 and ARM code generator and - // re-enable this optimization; otherwise, remove this TODO. - // b/26601270 - return false; - } if (index->IsConstant() || (index->IsBoundsCheck() && index->AsBoundsCheck()->GetIndex()->IsConstant())) { // When the index is a constant all the addressing can be fitted in the @@ -251,14 +242,20 @@ bool TryExtractArrayAccessAddress(HInstruction* access, // The access may require a runtime call or the original array pointer. return false; } + if (kEmitCompilerReadBarrier && + access->IsArrayGet() && + access->GetType() == Primitive::kPrimNot) { + // For object arrays, the read barrier instrumentation requires + // the original array pointer. + return false; + } // Proceed to extract the base address computation. HGraph* graph = access->GetBlock()->GetGraph(); ArenaAllocator* arena = graph->GetArena(); HIntConstant* offset = graph->GetIntConstant(data_offset); - HIntermediateAddress* address = - new (arena) HIntermediateAddress(array, offset, kNoDexPc); + HIntermediateAddress* address = new (arena) HIntermediateAddress(array, offset, kNoDexPc); // TODO: Is it ok to not have this on the intermediate address? // address->SetReferenceTypeInfo(array->GetReferenceTypeInfo()); access->GetBlock()->InsertInstructionBefore(address, access); diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc index 8790c1e4f1..93a2340a32 100644 --- a/compiler/optimizing/intrinsics_arm.cc +++ b/compiler/optimizing/intrinsics_arm.cc @@ -1945,7 +1945,7 @@ void IntrinsicCodeGeneratorARM::VisitSystemArrayCopy(HInvoke* invoke) { // if (src_ptr != end_ptr) { // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // // Slow-path copy. // do { @@ -1986,9 +1986,8 @@ void IntrinsicCodeGeneratorARM::VisitSystemArrayCopy(HInvoke* invoke) { // Given the numeric representation, it's enough to check the low bit of the // rb_state. We do that by shifting the bit out of the lock word with LSRS // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); __ Lsrs(temp2, temp2, LockWord::kReadBarrierStateShift + 1); // Carry flag is the last bit shifted out by LSRS. __ b(read_barrier_slow_path->GetEntryLabel(), CS); diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index db1c022868..47e6d9699d 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -2659,7 +2659,7 @@ void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) { // if (src_ptr != end_ptr) { // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // // Slow-path copy. // do { @@ -2704,9 +2704,8 @@ void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) { codegen_->AddSlowPath(read_barrier_slow_path); // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); __ Tbnz(tmp, LockWord::kReadBarrierStateShift, read_barrier_slow_path->GetEntryLabel()); // Fast-path copy. diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc new file mode 100644 index 0000000000..6ff0ca4eab --- /dev/null +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -0,0 +1,2694 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "intrinsics_arm_vixl.h" + +#include "arch/arm/instruction_set_features_arm.h" +#include "code_generator_arm_vixl.h" +#include "common_arm.h" +#include "lock_word.h" +#include "mirror/array-inl.h" + +#include "aarch32/constants-aarch32.h" + +namespace art { +namespace arm { + +#define __ assembler->GetVIXLAssembler()-> + +using helpers::DRegisterFrom; +using helpers::HighRegisterFrom; +using helpers::InputDRegisterAt; +using helpers::InputRegisterAt; +using helpers::InputSRegisterAt; +using helpers::InputVRegisterAt; +using helpers::Int32ConstantFrom; +using helpers::LocationFrom; +using helpers::LowRegisterFrom; +using helpers::LowSRegisterFrom; +using helpers::OutputDRegister; +using helpers::OutputRegister; +using helpers::OutputVRegister; +using helpers::RegisterFrom; +using helpers::SRegisterFrom; + +using namespace vixl::aarch32; // NOLINT(build/namespaces) + +ArmVIXLAssembler* IntrinsicCodeGeneratorARMVIXL::GetAssembler() { + return codegen_->GetAssembler(); +} + +ArenaAllocator* IntrinsicCodeGeneratorARMVIXL::GetAllocator() { + return codegen_->GetGraph()->GetArena(); +} + +// Default slow-path for fallback (calling the managed code to handle the intrinsic) in an +// intrinsified call. This will copy the arguments into the positions for a regular call. +// +// Note: The actual parameters are required to be in the locations given by the invoke's location +// summary. If an intrinsic modifies those locations before a slowpath call, they must be +// restored! +// +// Note: If an invoke wasn't sharpened, we will put down an invoke-virtual here. That's potentially +// sub-optimal (compared to a direct pointer call), but this is a slow-path. + +class IntrinsicSlowPathARMVIXL : public SlowPathCodeARMVIXL { + public: + explicit IntrinsicSlowPathARMVIXL(HInvoke* invoke) + : SlowPathCodeARMVIXL(invoke), invoke_(invoke) {} + + Location MoveArguments(CodeGenerator* codegen) { + InvokeDexCallingConventionVisitorARM calling_convention_visitor; + IntrinsicVisitor::MoveArguments(invoke_, codegen, &calling_convention_visitor); + return calling_convention_visitor.GetMethodLocation(); + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + ArmVIXLAssembler* assembler = down_cast<ArmVIXLAssembler*>(codegen->GetAssembler()); + __ Bind(GetEntryLabel()); + + SaveLiveRegisters(codegen, invoke_->GetLocations()); + + Location method_loc = MoveArguments(codegen); + + if (invoke_->IsInvokeStaticOrDirect()) { + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc); + } else { + codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc); + } + codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); + + // Copy the result back to the expected output. + Location out = invoke_->GetLocations()->Out(); + if (out.IsValid()) { + DCHECK(out.IsRegister()); // TODO: Replace this when we support output in memory. + DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg())); + codegen->MoveFromReturnRegister(out, invoke_->GetType()); + } + + RestoreLiveRegisters(codegen, invoke_->GetLocations()); + __ B(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPath"; } + + private: + // The instruction where this slow path is happening. + HInvoke* const invoke_; + + DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARMVIXL); +}; + +// Slow path implementing the SystemArrayCopy intrinsic copy loop with read barriers. +class ReadBarrierSystemArrayCopySlowPathARMVIXL : public SlowPathCodeARMVIXL { + public: + explicit ReadBarrierSystemArrayCopySlowPathARMVIXL(HInstruction* instruction) + : SlowPathCodeARMVIXL(instruction) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); + ArmVIXLAssembler* assembler = arm_codegen->GetAssembler(); + LocationSummary* locations = instruction_->GetLocations(); + DCHECK(locations->CanCall()); + DCHECK(instruction_->IsInvokeStaticOrDirect()) + << "Unexpected instruction in read barrier arraycopy slow path: " + << instruction_->DebugName(); + DCHECK(instruction_->GetLocations()->Intrinsified()); + DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy); + + int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot); + uint32_t element_size_shift = Primitive::ComponentSizeShift(Primitive::kPrimNot); + uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value(); + + vixl32::Register dest = InputRegisterAt(instruction_, 2); + Location dest_pos = locations->InAt(3); + vixl32::Register src_curr_addr = RegisterFrom(locations->GetTemp(0)); + vixl32::Register dst_curr_addr = RegisterFrom(locations->GetTemp(1)); + vixl32::Register src_stop_addr = RegisterFrom(locations->GetTemp(2)); + vixl32::Register tmp = RegisterFrom(locations->GetTemp(3)); + + __ Bind(GetEntryLabel()); + // Compute the base destination address in `dst_curr_addr`. + if (dest_pos.IsConstant()) { + int32_t constant = Int32ConstantFrom(dest_pos); + __ Add(dst_curr_addr, dest, element_size * constant + offset); + } else { + __ Add(dst_curr_addr, + dest, + Operand(RegisterFrom(dest_pos), vixl32::LSL, element_size_shift)); + __ Add(dst_curr_addr, dst_curr_addr, offset); + } + + vixl32::Label loop; + __ Bind(&loop); + __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex)); + assembler->MaybeUnpoisonHeapReference(tmp); + // TODO: Inline the mark bit check before calling the runtime? + // tmp = ReadBarrier::Mark(tmp); + // No need to save live registers; it's taken care of by the + // entrypoint. Also, there is no need to update the stack mask, + // as this runtime call will not trigger a garbage collection. + // (See ReadBarrierMarkSlowPathARM::EmitNativeCode for more + // explanations.) + DCHECK(!tmp.IsSP()); + DCHECK(!tmp.IsLR()); + DCHECK(!tmp.IsPC()); + // IP is used internally by the ReadBarrierMarkRegX entry point + // as a temporary (and not preserved). It thus cannot be used by + // any live register in this slow path. + DCHECK(!src_curr_addr.Is(ip)); + DCHECK(!dst_curr_addr.Is(ip)); + DCHECK(!src_stop_addr.Is(ip)); + DCHECK(!tmp.Is(ip)); + DCHECK(tmp.IsRegister()) << tmp; + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp.GetCode()); + // This runtime call does not require a stack map. + arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); + assembler->MaybePoisonHeapReference(tmp); + __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); + __ Cmp(src_curr_addr, src_stop_addr); + __ B(ne, &loop); + __ B(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { + return "ReadBarrierSystemArrayCopySlowPathARMVIXL"; + } + + private: + DISALLOW_COPY_AND_ASSIGN(ReadBarrierSystemArrayCopySlowPathARMVIXL); +}; + +IntrinsicLocationsBuilderARMVIXL::IntrinsicLocationsBuilderARMVIXL(CodeGeneratorARMVIXL* codegen) + : arena_(codegen->GetGraph()->GetArena()), + assembler_(codegen->GetAssembler()), + features_(codegen->GetInstructionSetFeatures()) {} + +bool IntrinsicLocationsBuilderARMVIXL::TryDispatch(HInvoke* invoke) { + Dispatch(invoke); + LocationSummary* res = invoke->GetLocations(); + if (res == nullptr) { + return false; + } + return res->Intrinsified(); +} + +static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresRegister()); +} + +static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); +} + +static void MoveFPToInt(LocationSummary* locations, bool is64bit, ArmVIXLAssembler* assembler) { + Location input = locations->InAt(0); + Location output = locations->Out(); + if (is64bit) { + __ Vmov(LowRegisterFrom(output), HighRegisterFrom(output), DRegisterFrom(input)); + } else { + __ Vmov(RegisterFrom(output), SRegisterFrom(input)); + } +} + +static void MoveIntToFP(LocationSummary* locations, bool is64bit, ArmVIXLAssembler* assembler) { + Location input = locations->InAt(0); + Location output = locations->Out(); + if (is64bit) { + __ Vmov(DRegisterFrom(output), LowRegisterFrom(input), HighRegisterFrom(input)); + } else { + __ Vmov(SRegisterFrom(output), RegisterFrom(input)); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitDoubleLongBitsToDouble(HInvoke* invoke) { + CreateIntToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { + MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); +} +void IntrinsicCodeGeneratorARMVIXL::VisitDoubleLongBitsToDouble(HInvoke* invoke) { + MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitFloatFloatToRawIntBits(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitFloatIntBitsToFloat(HInvoke* invoke) { + CreateIntToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitFloatFloatToRawIntBits(HInvoke* invoke) { + MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); +} +void IntrinsicCodeGeneratorARMVIXL::VisitFloatIntBitsToFloat(HInvoke* invoke) { + MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); +} + +static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); +} + +static void GenNumberOfLeadingZeros(LocationSummary* locations, + Primitive::Type type, + ArmVIXLAssembler* assembler) { + Location in = locations->InAt(0); + vixl32::Register out = RegisterFrom(locations->Out()); + + DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); + + if (type == Primitive::kPrimLong) { + vixl32::Register in_reg_lo = LowRegisterFrom(in); + vixl32::Register in_reg_hi = HighRegisterFrom(in); + vixl32::Label end; + __ Clz(out, in_reg_hi); + __ Cbnz(in_reg_hi, &end); + __ Clz(out, in_reg_lo); + __ Add(out, out, 32); + __ Bind(&end); + } else { + __ Clz(out, RegisterFrom(in)); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { + GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { + GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler()); +} + +static void GenNumberOfTrailingZeros(LocationSummary* locations, + Primitive::Type type, + ArmVIXLAssembler* assembler) { + DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); + + vixl32::Register out = RegisterFrom(locations->Out()); + + if (type == Primitive::kPrimLong) { + vixl32::Register in_reg_lo = LowRegisterFrom(locations->InAt(0)); + vixl32::Register in_reg_hi = HighRegisterFrom(locations->InAt(0)); + vixl32::Label end; + __ Rbit(out, in_reg_lo); + __ Clz(out, out); + __ Cbnz(in_reg_lo, &end); + __ Rbit(out, in_reg_hi); + __ Clz(out, out); + __ Add(out, out, 32); + __ Bind(&end); + } else { + vixl32::Register in = RegisterFrom(locations->InAt(0)); + __ Rbit(out, in); + __ Clz(out, out); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { + GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { + GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler()); +} + +static void MathAbsFP(HInvoke* invoke, ArmVIXLAssembler* assembler) { + __ Vabs(OutputVRegister(invoke), InputVRegisterAt(invoke, 0)); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsDouble(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsDouble(HInvoke* invoke) { + MathAbsFP(invoke, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsFloat(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsFloat(HInvoke* invoke) { + MathAbsFP(invoke, GetAssembler()); +} + +static void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + + locations->AddTemp(Location::RequiresRegister()); +} + +static void GenAbsInteger(LocationSummary* locations, + bool is64bit, + ArmVIXLAssembler* assembler) { + Location in = locations->InAt(0); + Location output = locations->Out(); + + vixl32::Register mask = RegisterFrom(locations->GetTemp(0)); + + if (is64bit) { + vixl32::Register in_reg_lo = LowRegisterFrom(in); + vixl32::Register in_reg_hi = HighRegisterFrom(in); + vixl32::Register out_reg_lo = LowRegisterFrom(output); + vixl32::Register out_reg_hi = HighRegisterFrom(output); + + DCHECK(!out_reg_lo.Is(in_reg_hi)) << "Diagonal overlap unexpected."; + + __ Asr(mask, in_reg_hi, 31); + __ Adds(out_reg_lo, in_reg_lo, mask); + __ Adc(out_reg_hi, in_reg_hi, mask); + __ Eor(out_reg_lo, mask, out_reg_lo); + __ Eor(out_reg_hi, mask, out_reg_hi); + } else { + vixl32::Register in_reg = RegisterFrom(in); + vixl32::Register out_reg = RegisterFrom(output); + + __ Asr(mask, in_reg, 31); + __ Add(out_reg, in_reg, mask); + __ Eor(out_reg, mask, out_reg); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsInt(HInvoke* invoke) { + CreateIntToIntPlusTemp(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsInt(HInvoke* invoke) { + GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); +} + + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsLong(HInvoke* invoke) { + CreateIntToIntPlusTemp(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsLong(HInvoke* invoke) { + GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); +} + +static void GenMinMax(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) { + vixl32::Register op1 = InputRegisterAt(invoke, 0); + vixl32::Register op2 = InputRegisterAt(invoke, 1); + vixl32::Register out = OutputRegister(invoke); + + __ Cmp(op1, op2); + + { + AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + + __ ite(is_min ? lt : gt); + __ mov(is_min ? lt : gt, out, op1); + __ mov(is_min ? ge : le, out, op2); + } +} + +static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMinIntInt(HInvoke* invoke) { + CreateIntIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMinIntInt(HInvoke* invoke) { + GenMinMax(invoke, /* is_min */ true, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxIntInt(HInvoke* invoke) { + CreateIntIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxIntInt(HInvoke* invoke) { + GenMinMax(invoke, /* is_min */ false, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathSqrt(HInvoke* invoke) { + CreateFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathSqrt(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Vsqrt(OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPeekByte(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPeekByte(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + // Ignore upper 4B of long address. + __ Ldrsb(OutputRegister(invoke), LowRegisterFrom(invoke->GetLocations()->InAt(0))); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPeekIntNative(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPeekIntNative(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + // Ignore upper 4B of long address. + __ Ldr(OutputRegister(invoke), LowRegisterFrom(invoke->GetLocations()->InAt(0))); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPeekLongNative(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPeekLongNative(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + // Ignore upper 4B of long address. + vixl32::Register addr = LowRegisterFrom(invoke->GetLocations()->InAt(0)); + // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor + // exception. So we can't use ldrd as addr may be unaligned. + vixl32::Register lo = LowRegisterFrom(invoke->GetLocations()->Out()); + vixl32::Register hi = HighRegisterFrom(invoke->GetLocations()->Out()); + if (addr.Is(lo)) { + __ Ldr(hi, MemOperand(addr, 4)); + __ Ldr(lo, addr); + } else { + __ Ldr(lo, addr); + __ Ldr(hi, MemOperand(addr, 4)); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPeekShortNative(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPeekShortNative(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + // Ignore upper 4B of long address. + __ Ldrsh(OutputRegister(invoke), LowRegisterFrom(invoke->GetLocations()->InAt(0))); +} + +static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPokeByte(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPokeByte(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Strb(InputRegisterAt(invoke, 1), LowRegisterFrom(invoke->GetLocations()->InAt(0))); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPokeIntNative(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPokeIntNative(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Str(InputRegisterAt(invoke, 1), LowRegisterFrom(invoke->GetLocations()->InAt(0))); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPokeLongNative(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPokeLongNative(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + // Ignore upper 4B of long address. + vixl32::Register addr = LowRegisterFrom(invoke->GetLocations()->InAt(0)); + // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor + // exception. So we can't use ldrd as addr may be unaligned. + __ Str(LowRegisterFrom(invoke->GetLocations()->InAt(1)), addr); + __ Str(HighRegisterFrom(invoke->GetLocations()->InAt(1)), MemOperand(addr, 4)); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPokeShortNative(HInvoke* invoke) { + CreateIntIntToVoidLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMemoryPokeShortNative(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Strh(InputRegisterAt(invoke, 1), LowRegisterFrom(invoke->GetLocations()->InAt(0))); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitThreadCurrentThread(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetOut(Location::RequiresRegister()); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitThreadCurrentThread(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Ldr(OutputRegister(invoke), + MemOperand(tr, Thread::PeerOffset<kArmPointerSize>().Int32Value())); +} + +static void GenUnsafeGet(HInvoke* invoke, + Primitive::Type type, + bool is_volatile, + CodeGeneratorARMVIXL* codegen) { + LocationSummary* locations = invoke->GetLocations(); + ArmVIXLAssembler* assembler = codegen->GetAssembler(); + Location base_loc = locations->InAt(1); + vixl32::Register base = InputRegisterAt(invoke, 1); // Object pointer. + Location offset_loc = locations->InAt(2); + vixl32::Register offset = LowRegisterFrom(offset_loc); // Long offset, lo part only. + Location trg_loc = locations->Out(); + + switch (type) { + case Primitive::kPrimInt: { + vixl32::Register trg = RegisterFrom(trg_loc); + __ Ldr(trg, MemOperand(base, offset)); + if (is_volatile) { + __ Dmb(vixl32::ISH); + } + break; + } + + case Primitive::kPrimNot: { + vixl32::Register trg = RegisterFrom(trg_loc); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + codegen->GenerateReferenceLoadWithBakerReadBarrier( + invoke, trg_loc, base, 0U, offset_loc, TIMES_1, temp, /* needs_null_check */ false); + if (is_volatile) { + __ Dmb(vixl32::ISH); + } + } else { + __ Ldr(trg, MemOperand(base, offset)); + if (is_volatile) { + __ Dmb(vixl32::ISH); + } + codegen->GenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc); + } + } else { + __ Ldr(trg, MemOperand(base, offset)); + if (is_volatile) { + __ Dmb(vixl32::ISH); + } + assembler->MaybeUnpoisonHeapReference(trg); + } + break; + } + + case Primitive::kPrimLong: { + vixl32::Register trg_lo = LowRegisterFrom(trg_loc); + vixl32::Register trg_hi = HighRegisterFrom(trg_loc); + if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { + __ Ldrexd(trg_lo, trg_hi, MemOperand(base, offset)); + } else { + __ Ldrd(trg_lo, trg_hi, MemOperand(base, offset)); + } + if (is_volatile) { + __ Dmb(vixl32::ISH); + } + break; + } + + default: + LOG(FATAL) << "Unexpected type " << type; + UNREACHABLE(); + } +} + +static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, + HInvoke* invoke, + Primitive::Type type) { + bool can_call = kEmitCompilerReadBarrier && + (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || + invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); + LocationSummary* locations = new (arena) LocationSummary(invoke, + (can_call + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall), + kIntrinsified); + if (can_call && kUseBakerReadBarrier) { + locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + } + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), + (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); + if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in InstructionCodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeGet(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeGetVolatile(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeGetLong(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeGetLongVolatile(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeGetObject(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeGet(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeGetVolatile(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeGetLong(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeGetLongVolatile(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeGetObject(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { + GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_); +} + +static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, + const ArmInstructionSetFeatures& features, + Primitive::Type type, + bool is_volatile, + HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetInAt(3, Location::RequiresRegister()); + + if (type == Primitive::kPrimLong) { + // Potentially need temps for ldrexd-strexd loop. + if (is_volatile && !features.HasAtomicLdrdAndStrd()) { + locations->AddTemp(Location::RequiresRegister()); // Temp_lo. + locations->AddTemp(Location::RequiresRegister()); // Temp_hi. + } + } else if (type == Primitive::kPrimNot) { + // Temps for card-marking. + locations->AddTemp(Location::RequiresRegister()); // Temp. + locations->AddTemp(Location::RequiresRegister()); // Card. + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePut(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutOrdered(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutVolatile(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ true, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutObject(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutObjectOrdered(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutObjectVolatile(HInvoke* invoke) { + CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ true, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutLong(HInvoke* invoke) { + CreateIntIntIntIntToVoid( + arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutLongOrdered(HInvoke* invoke) { + CreateIntIntIntIntToVoid( + arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafePutLongVolatile(HInvoke* invoke) { + CreateIntIntIntIntToVoid( + arena_, features_, Primitive::kPrimLong, /* is_volatile */ true, invoke); +} + +static void GenUnsafePut(LocationSummary* locations, + Primitive::Type type, + bool is_volatile, + bool is_ordered, + CodeGeneratorARMVIXL* codegen) { + ArmVIXLAssembler* assembler = codegen->GetAssembler(); + + vixl32::Register base = RegisterFrom(locations->InAt(1)); // Object pointer. + vixl32::Register offset = LowRegisterFrom(locations->InAt(2)); // Long offset, lo part only. + vixl32::Register value; + + if (is_volatile || is_ordered) { + __ Dmb(vixl32::ISH); + } + + if (type == Primitive::kPrimLong) { + vixl32::Register value_lo = LowRegisterFrom(locations->InAt(3)); + vixl32::Register value_hi = HighRegisterFrom(locations->InAt(3)); + value = value_lo; + if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { + vixl32::Register temp_lo = RegisterFrom(locations->GetTemp(0)); + vixl32::Register temp_hi = RegisterFrom(locations->GetTemp(1)); + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + const vixl32::Register temp_reg = temps.Acquire(); + + __ Add(temp_reg, base, offset); + vixl32::Label loop_head; + __ Bind(&loop_head); + __ Ldrexd(temp_lo, temp_hi, temp_reg); + __ Strexd(temp_lo, value_lo, value_hi, temp_reg); + __ Cmp(temp_lo, 0); + __ B(ne, &loop_head); + } else { + __ Strd(value_lo, value_hi, MemOperand(base, offset)); + } + } else { + value = RegisterFrom(locations->InAt(3)); + vixl32::Register source = value; + if (kPoisonHeapReferences && type == Primitive::kPrimNot) { + vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + __ Mov(temp, value); + assembler->PoisonHeapReference(temp); + source = temp; + } + __ Str(source, MemOperand(base, offset)); + } + + if (is_volatile) { + __ Dmb(vixl32::ISH); + } + + if (type == Primitive::kPrimNot) { + vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + vixl32::Register card = RegisterFrom(locations->GetTemp(1)); + bool value_can_be_null = true; // TODO: Worth finding out this information? + codegen->MarkGCCard(temp, card, base, value, value_can_be_null); + } +} + +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePut(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimInt, + /* is_volatile */ false, + /* is_ordered */ false, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutOrdered(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimInt, + /* is_volatile */ false, + /* is_ordered */ true, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutVolatile(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimInt, + /* is_volatile */ true, + /* is_ordered */ false, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutObject(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimNot, + /* is_volatile */ false, + /* is_ordered */ false, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutObjectOrdered(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimNot, + /* is_volatile */ false, + /* is_ordered */ true, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutObjectVolatile(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimNot, + /* is_volatile */ true, + /* is_ordered */ false, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutLong(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimLong, + /* is_volatile */ false, + /* is_ordered */ false, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutLongOrdered(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimLong, + /* is_volatile */ false, + /* is_ordered */ true, + codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafePutLongVolatile(HInvoke* invoke) { + GenUnsafePut(invoke->GetLocations(), + Primitive::kPrimLong, + /* is_volatile */ true, + /* is_ordered */ false, + codegen_); +} + +static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena, + HInvoke* invoke, + Primitive::Type type) { + bool can_call = kEmitCompilerReadBarrier && + kUseBakerReadBarrier && + (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject); + LocationSummary* locations = new (arena) LocationSummary(invoke, + (can_call + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall), + kIntrinsified); + locations->SetInAt(0, Location::NoLocation()); // Unused receiver. + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetInAt(3, Location::RequiresRegister()); + locations->SetInAt(4, Location::RequiresRegister()); + + // If heap poisoning is enabled, we don't want the unpoisoning + // operations to potentially clobber the output. Likewise when + // emitting a (Baker) read barrier, which may call. + Location::OutputOverlap overlaps = + ((kPoisonHeapReferences && type == Primitive::kPrimNot) || can_call) + ? Location::kOutputOverlap + : Location::kNoOutputOverlap; + locations->SetOut(Location::RequiresRegister(), overlaps); + + // Temporary registers used in CAS. In the object case + // (UnsafeCASObject intrinsic), these are also used for + // card-marking, and possibly for (Baker) read barrier. + locations->AddTemp(Location::RequiresRegister()); // Pointer. + locations->AddTemp(Location::RequiresRegister()); // Temp 1. +} + +static void GenCas(HInvoke* invoke, Primitive::Type type, CodeGeneratorARMVIXL* codegen) { + DCHECK_NE(type, Primitive::kPrimLong); + + ArmVIXLAssembler* assembler = codegen->GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + Location out_loc = locations->Out(); + vixl32::Register out = OutputRegister(invoke); // Boolean result. + + vixl32::Register base = InputRegisterAt(invoke, 1); // Object pointer. + Location offset_loc = locations->InAt(2); + vixl32::Register offset = LowRegisterFrom(offset_loc); // Offset (discard high 4B). + vixl32::Register expected = InputRegisterAt(invoke, 3); // Expected. + vixl32::Register value = InputRegisterAt(invoke, 4); // Value. + + Location tmp_ptr_loc = locations->GetTemp(0); + vixl32::Register tmp_ptr = RegisterFrom(tmp_ptr_loc); // Pointer to actual memory. + vixl32::Register tmp = RegisterFrom(locations->GetTemp(1)); // Value in memory. + + if (type == Primitive::kPrimNot) { + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); + + // Mark card for object assuming new value is stored. Worst case we will mark an unchanged + // object and scan the receiver at the next GC for nothing. + bool value_can_be_null = true; // TODO: Worth finding out this information? + codegen->MarkGCCard(tmp_ptr, tmp, base, value, value_can_be_null); + + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Need to make sure the reference stored in the field is a to-space + // one before attempting the CAS or the CAS could fail incorrectly. + codegen->GenerateReferenceLoadWithBakerReadBarrier( + invoke, + out_loc, // Unused, used only as a "temporary" within the read barrier. + base, + /* offset */ 0u, + /* index */ offset_loc, + ScaleFactor::TIMES_1, + tmp_ptr_loc, + /* needs_null_check */ false, + /* always_update_field */ true, + &tmp); + } + } + + // Prevent reordering with prior memory operations. + // Emit a DMB ISH instruction instead of an DMB ISHST one, as the + // latter allows a preceding load to be delayed past the STXR + // instruction below. + __ Dmb(vixl32::ISH); + + __ Add(tmp_ptr, base, offset); + + if (kPoisonHeapReferences && type == Primitive::kPrimNot) { + codegen->GetAssembler()->PoisonHeapReference(expected); + if (value.Is(expected)) { + // Do not poison `value`, as it is the same register as + // `expected`, which has just been poisoned. + } else { + codegen->GetAssembler()->PoisonHeapReference(value); + } + } + + // do { + // tmp = [r_ptr] - expected; + // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); + // result = tmp != 0; + + vixl32::Label loop_head; + __ Bind(&loop_head); + + __ Ldrex(tmp, tmp_ptr); + + __ Subs(tmp, tmp, expected); + + { + AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + + __ itt(eq); + __ strex(eq, tmp, value, tmp_ptr); + __ cmp(eq, tmp, 1); + } + + __ B(eq, &loop_head); + + __ Dmb(vixl32::ISH); + + __ Rsbs(out, tmp, 1); + + { + AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + + __ it(cc); + __ mov(cc, out, 0); + } + + if (kPoisonHeapReferences && type == Primitive::kPrimNot) { + codegen->GetAssembler()->UnpoisonHeapReference(expected); + if (value.Is(expected)) { + // Do not unpoison `value`, as it is the same register as + // `expected`, which has just been unpoisoned. + } else { + codegen->GetAssembler()->UnpoisonHeapReference(value); + } + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeCASInt(HInvoke* invoke) { + CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimInt); +} +void IntrinsicLocationsBuilderARMVIXL::VisitUnsafeCASObject(HInvoke* invoke) { + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { + return; + } + + CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimNot); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeCASInt(HInvoke* invoke) { + GenCas(invoke, Primitive::kPrimInt, codegen_); +} +void IntrinsicCodeGeneratorARMVIXL::VisitUnsafeCASObject(HInvoke* invoke) { + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); + + GenCas(invoke, Primitive::kPrimNot, codegen_); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringCompareTo(HInvoke* invoke) { + // The inputs plus one temp. + LocationSummary* locations = new (arena_) LocationSummary(invoke, + invoke->InputAt(1)->CanBeNull() + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + // Need temporary registers for String compression's feature. + if (mirror::kUseStringCompression) { + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + } + locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + vixl32::Register str = InputRegisterAt(invoke, 0); + vixl32::Register arg = InputRegisterAt(invoke, 1); + vixl32::Register out = OutputRegister(invoke); + + vixl32::Register temp0 = RegisterFrom(locations->GetTemp(0)); + vixl32::Register temp1 = RegisterFrom(locations->GetTemp(1)); + vixl32::Register temp2 = RegisterFrom(locations->GetTemp(2)); + vixl32::Register temp3, temp4; + if (mirror::kUseStringCompression) { + temp3 = RegisterFrom(locations->GetTemp(3)); + temp4 = RegisterFrom(locations->GetTemp(4)); + } + + vixl32::Label loop; + vixl32::Label find_char_diff; + vixl32::Label end; + vixl32::Label different_compression; + + // Get offsets of count and value fields within a string object. + const int32_t count_offset = mirror::String::CountOffset().Int32Value(); + const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); + + // Note that the null check must have been done earlier. + DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); + + // Take slow path and throw if input can be and is null. + SlowPathCodeARMVIXL* slow_path = nullptr; + const bool can_slow_path = invoke->InputAt(1)->CanBeNull(); + if (can_slow_path) { + slow_path = new (GetAllocator()) IntrinsicSlowPathARMVIXL(invoke); + codegen_->AddSlowPath(slow_path); + __ Cbz(arg, slow_path->GetEntryLabel()); + } + + // Reference equality check, return 0 if same reference. + __ Subs(out, str, arg); + __ B(eq, &end); + + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + vixl32::Register temp_reg = temps.Acquire(); + + if (mirror::kUseStringCompression) { + // Load lengths of this and argument strings. + __ Ldr(temp3, MemOperand(str, count_offset)); + __ Ldr(temp4, MemOperand(arg, count_offset)); + // Clean out compression flag from lengths. + __ Bic(temp0, temp3, 0x80000000); + __ Bic(temp_reg, temp4, 0x80000000); + } else { + // Load lengths of this and argument strings. + __ Ldr(temp0, MemOperand(str, count_offset)); + __ Ldr(temp_reg, MemOperand(arg, count_offset)); + } + // out = length diff. + __ Subs(out, temp0, temp_reg); + // temp0 = min(len(str), len(arg)). + + { + AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + + __ it(gt); + __ mov(gt, temp0, temp_reg); + } + + temps.Release(temp_reg); + // Shorter string is empty? + __ Cbz(temp0, &end); + + if (mirror::kUseStringCompression) { + // Check if both strings using same compression style to use this comparison loop. + __ Eors(temp3, temp3, temp4); + __ B(mi, &different_compression); + } + // Store offset of string value in preparation for comparison loop. + __ Mov(temp1, value_offset); + if (mirror::kUseStringCompression) { + // For string compression, calculate the number of bytes to compare (not chars). + // This could in theory exceed INT32_MAX, so treat temp0 as unsigned. + __ Cmp(temp4, 0); + + AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + + __ it(ge); + __ add(ge, temp0, temp0, temp0); + } + + // Assertions that must hold in order to compare multiple characters at a time. + CHECK_ALIGNED(value_offset, 8); + static_assert(IsAligned<8>(kObjectAlignment), + "String data must be 8-byte aligned for unrolled CompareTo loop."); + + const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); + DCHECK_EQ(char_size, 2u); + + vixl32::Label find_char_diff_2nd_cmp; + // Unrolled loop comparing 4x16-bit chars per iteration (ok because of string data alignment). + __ Bind(&loop); + temp_reg = temps.Acquire(); + __ Ldr(temp_reg, MemOperand(str, temp1)); + __ Ldr(temp2, MemOperand(arg, temp1)); + __ Cmp(temp_reg, temp2); + __ B(ne, &find_char_diff); + __ Add(temp1, temp1, char_size * 2); + + __ Ldr(temp_reg, MemOperand(str, temp1)); + __ Ldr(temp2, MemOperand(arg, temp1)); + __ Cmp(temp_reg, temp2); + __ B(ne, &find_char_diff_2nd_cmp); + __ Add(temp1, temp1, char_size * 2); + // With string compression, we have compared 8 bytes, otherwise 4 chars. + __ Subs(temp0, temp0, (mirror::kUseStringCompression ? 8 : 4)); + __ B(hi, &loop); + __ B(&end); + + __ Bind(&find_char_diff_2nd_cmp); + if (mirror::kUseStringCompression) { + __ Subs(temp0, temp0, 4); // 4 bytes previously compared. + __ B(ls, &end); // Was the second comparison fully beyond the end? + } else { + // Without string compression, we can start treating temp0 as signed + // and rely on the signed comparison below. + __ Sub(temp0, temp0, 2); + } + + // Find the single character difference. + __ Bind(&find_char_diff); + // Get the bit position of the first character that differs. + __ Eor(temp1, temp2, temp_reg); + __ Rbit(temp1, temp1); + __ Clz(temp1, temp1); + + // temp0 = number of characters remaining to compare. + // (Without string compression, it could be < 1 if a difference is found by the second CMP + // in the comparison loop, and after the end of the shorter string data). + + // Without string compression (temp1 >> 4) = character where difference occurs between the last + // two words compared, in the interval [0,1]. + // (0 for low half-word different, 1 for high half-word different). + // With string compression, (temp1 << 3) = byte where the difference occurs, + // in the interval [0,3]. + + // If temp0 <= (temp1 >> (kUseStringCompression ? 3 : 4)), the difference occurs outside + // the remaining string data, so just return length diff (out). + // The comparison is unsigned for string compression, otherwise signed. + __ Cmp(temp0, Operand(temp1, vixl32::LSR, (mirror::kUseStringCompression ? 3 : 4))); + __ B((mirror::kUseStringCompression ? ls : le), &end); + // Extract the characters and calculate the difference. + vixl32::Label uncompressed_string, continue_process; + if (mirror::kUseStringCompression) { + __ Cmp(temp4, 0); + __ B(ge, &uncompressed_string); + __ Bic(temp1, temp1, 0x7); + __ B(&continue_process); + } + __ Bind(&uncompressed_string); + __ Bic(temp1, temp1, 0xf); + __ Bind(&continue_process); + + __ Lsr(temp2, temp2, temp1); + __ Lsr(temp_reg, temp_reg, temp1); + vixl32::Label calculate_difference, uncompressed_string_extract_chars; + if (mirror::kUseStringCompression) { + __ Cmp(temp4, 0); + __ B(ge, &uncompressed_string_extract_chars); + __ Ubfx(temp2, temp2, 0, 8); + __ Ubfx(temp_reg, temp_reg, 0, 8); + __ B(&calculate_difference); + } + __ Bind(&uncompressed_string_extract_chars); + __ Movt(temp2, 0); + __ Movt(temp_reg, 0); + __ Bind(&calculate_difference); + __ Sub(out, temp_reg, temp2); + temps.Release(temp_reg); + __ B(&end); + + if (mirror::kUseStringCompression) { + const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte); + DCHECK_EQ(c_char_size, 1u); + vixl32::Label loop_arg_compressed, loop_this_compressed, find_diff; + // Comparison for different compression style. + // This part is when THIS is compressed and ARG is not. + __ Bind(&different_compression); + __ Add(temp2, str, value_offset); + __ Add(temp3, arg, value_offset); + __ Cmp(temp4, 0); + __ B(lt, &loop_arg_compressed); + + __ Bind(&loop_this_compressed); + temp_reg = temps.Acquire(); + __ Ldrb(temp_reg, MemOperand(temp2, c_char_size, PostIndex)); + __ Ldrh(temp4, MemOperand(temp3, char_size, PostIndex)); + __ Cmp(temp_reg, temp4); + __ B(ne, &find_diff); + __ Subs(temp0, temp0, 1); + __ B(gt, &loop_this_compressed); + __ B(&end); + + // This part is when THIS is not compressed and ARG is. + __ Bind(&loop_arg_compressed); + __ Ldrh(temp_reg, MemOperand(temp2, char_size, PostIndex)); + __ Ldrb(temp4, MemOperand(temp3, c_char_size, PostIndex)); + __ Cmp(temp_reg, temp4); + __ B(ne, &find_diff); + __ Subs(temp0, temp0, 1); + __ B(gt, &loop_arg_compressed); + __ B(&end); + + // Calculate the difference. + __ Bind(&find_diff); + __ Sub(out, temp_reg, temp4); + temps.Release(temp_reg); + } + + __ Bind(&end); + + if (can_slow_path) { + __ Bind(slow_path->GetExitLabel()); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringEquals(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + InvokeRuntimeCallingConventionARMVIXL calling_convention; + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + // Temporary registers to store lengths of strings and for calculations. + // Using instruction cbz requires a low register, so explicitly set a temp to be R0. + locations->AddTemp(LocationFrom(r0)); + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + + locations->SetOut(Location::RequiresRegister()); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringEquals(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + vixl32::Register str = InputRegisterAt(invoke, 0); + vixl32::Register arg = InputRegisterAt(invoke, 1); + vixl32::Register out = OutputRegister(invoke); + + vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + vixl32::Register temp1 = RegisterFrom(locations->GetTemp(1)); + vixl32::Register temp2 = RegisterFrom(locations->GetTemp(2)); + + vixl32::Label loop, preloop; + vixl32::Label end; + vixl32::Label return_true; + vixl32::Label return_false; + + // Get offsets of count, value, and class fields within a string object. + const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); + const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); + const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value(); + + // Note that the null check must have been done earlier. + DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); + + StringEqualsOptimizations optimizations(invoke); + if (!optimizations.GetArgumentNotNull()) { + // Check if input is null, return false if it is. + __ Cbz(arg, &return_false); + } + + if (!optimizations.GetArgumentIsString()) { + // Instanceof check for the argument by comparing class fields. + // All string objects must have the same type since String cannot be subclassed. + // Receiver must be a string object, so its class field is equal to all strings' class fields. + // If the argument is a string object, its class field must be equal to receiver's class field. + __ Ldr(temp, MemOperand(str, class_offset)); + __ Ldr(temp1, MemOperand(arg, class_offset)); + __ Cmp(temp, temp1); + __ B(ne, &return_false); + } + + // Load lengths of this and argument strings. + __ Ldr(temp, MemOperand(str, count_offset)); + __ Ldr(temp1, MemOperand(arg, count_offset)); + // Check if lengths are equal, return false if they're not. + // Also compares the compression style, if differs return false. + __ Cmp(temp, temp1); + __ B(ne, &return_false); + // Return true if both strings are empty. + if (mirror::kUseStringCompression) { + // Length needs to be masked out first because 0 is treated as compressed. + __ Bic(temp, temp, 0x80000000); + } + __ Cbz(temp, &return_true); + // Reference equality check, return true if same reference. + __ Cmp(str, arg); + __ B(eq, &return_true); + + // Assertions that must hold in order to compare strings 2 characters at a time. + DCHECK_ALIGNED(value_offset, 4); + static_assert(IsAligned<4>(kObjectAlignment), "String data must be aligned for fast compare."); + + if (mirror::kUseStringCompression) { + // If not compressed, directly to fast compare. Else do preprocess on length. + __ Cmp(temp1, 0); + __ B(gt, &preloop); + // Mask out compression flag and adjust length for compressed string (8-bit) + // as if it is a 16-bit data, new_length = (length + 1) / 2. + __ Add(temp, temp, 1); + __ Lsr(temp, temp, 1); + __ Bind(&preloop); + } + // Loop to compare strings 2 characters at a time starting at the front of the string. + // Ok to do this because strings with an odd length are zero-padded. + __ Mov(temp1, value_offset); + __ Bind(&loop); + __ Ldr(out, MemOperand(str, temp1)); + __ Ldr(temp2, MemOperand(arg, temp1)); + __ Cmp(out, temp2); + __ B(ne, &return_false); + __ Add(temp1, temp1, sizeof(uint32_t)); + __ Subs(temp, temp, sizeof(uint32_t) / sizeof(uint16_t)); + __ B(gt, &loop); + + // Return true and exit the function. + // If loop does not result in returning false, we return true. + __ Bind(&return_true); + __ Mov(out, 1); + __ B(&end); + + // Return false and exit the function. + __ Bind(&return_false); + __ Mov(out, 0); + __ Bind(&end); +} + +static void GenerateVisitStringIndexOf(HInvoke* invoke, + ArmVIXLAssembler* assembler, + CodeGeneratorARMVIXL* codegen, + ArenaAllocator* allocator, + bool start_at_zero) { + LocationSummary* locations = invoke->GetLocations(); + + // Note that the null check must have been done earlier. + DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); + + // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, + // or directly dispatch for a large constant, or omit slow-path for a small constant or a char. + SlowPathCodeARMVIXL* slow_path = nullptr; + HInstruction* code_point = invoke->InputAt(1); + if (code_point->IsIntConstant()) { + if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > + std::numeric_limits<uint16_t>::max()) { + // Always needs the slow-path. We could directly dispatch to it, but this case should be + // rare, so for simplicity just put the full slow-path down and branch unconditionally. + slow_path = new (allocator) IntrinsicSlowPathARMVIXL(invoke); + codegen->AddSlowPath(slow_path); + __ B(slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); + return; + } + } else if (code_point->GetType() != Primitive::kPrimChar) { + vixl32::Register char_reg = InputRegisterAt(invoke, 1); + // 0xffff is not modified immediate but 0x10000 is, so use `>= 0x10000` instead of `> 0xffff`. + __ Cmp(char_reg, static_cast<uint32_t>(std::numeric_limits<uint16_t>::max()) + 1); + slow_path = new (allocator) IntrinsicSlowPathARMVIXL(invoke); + codegen->AddSlowPath(slow_path); + __ B(hs, slow_path->GetEntryLabel()); + } + + if (start_at_zero) { + vixl32::Register tmp_reg = RegisterFrom(locations->GetTemp(0)); + DCHECK(tmp_reg.Is(r2)); + // Start-index = 0. + __ Mov(tmp_reg, 0); + } + + codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path); + CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>(); + + if (slow_path != nullptr) { + __ Bind(slow_path->GetExitLabel()); + } +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringIndexOf(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kCallOnMainAndSlowPath, + kIntrinsified); + // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's + // best to align the inputs accordingly. + InvokeRuntimeCallingConventionARMVIXL calling_convention; + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); + locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); + locations->SetOut(LocationFrom(r0)); + + // Need to send start-index=0. + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2))); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringIndexOf(HInvoke* invoke) { + GenerateVisitStringIndexOf( + invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringIndexOfAfter(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kCallOnMainAndSlowPath, + kIntrinsified); + // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's + // best to align the inputs accordingly. + InvokeRuntimeCallingConventionARMVIXL calling_convention; + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); + locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); + locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); + locations->SetOut(LocationFrom(r0)); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringIndexOfAfter(HInvoke* invoke) { + GenerateVisitStringIndexOf( + invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringNewStringFromBytes(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kCallOnMainAndSlowPath, + kIntrinsified); + InvokeRuntimeCallingConventionARMVIXL calling_convention; + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); + locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); + locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); + locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3))); + locations->SetOut(LocationFrom(r0)); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringNewStringFromBytes(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + vixl32::Register byte_array = InputRegisterAt(invoke, 0); + __ Cmp(byte_array, 0); + SlowPathCodeARMVIXL* slow_path = new (GetAllocator()) IntrinsicSlowPathARMVIXL(invoke); + codegen_->AddSlowPath(slow_path); + __ B(eq, slow_path->GetEntryLabel()); + + codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc(), slow_path); + CheckEntrypointTypes<kQuickAllocStringFromBytes, void*, void*, int32_t, int32_t, int32_t>(); + __ Bind(slow_path->GetExitLabel()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringNewStringFromChars(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kCallOnMainOnly, + kIntrinsified); + InvokeRuntimeCallingConventionARMVIXL calling_convention; + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); + locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); + locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); + locations->SetOut(LocationFrom(r0)); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringNewStringFromChars(HInvoke* invoke) { + // No need to emit code checking whether `locations->InAt(2)` is a null + // pointer, as callers of the native method + // + // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) + // + // all include a null check on `data` before calling that method. + codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); + CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>(); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringNewStringFromString(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kCallOnMainAndSlowPath, + kIntrinsified); + InvokeRuntimeCallingConventionARMVIXL calling_convention; + locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); + locations->SetOut(LocationFrom(r0)); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringNewStringFromString(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + vixl32::Register string_to_copy = InputRegisterAt(invoke, 0); + __ Cmp(string_to_copy, 0); + SlowPathCodeARMVIXL* slow_path = new (GetAllocator()) IntrinsicSlowPathARMVIXL(invoke); + codegen_->AddSlowPath(slow_path); + __ B(eq, slow_path->GetEntryLabel()); + + codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc(), slow_path); + CheckEntrypointTypes<kQuickAllocStringFromString, void*, void*>(); + + __ Bind(slow_path->GetExitLabel()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { + // The only read barrier implementation supporting the + // SystemArrayCopy intrinsic is the Baker-style read barriers. + if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { + return; + } + + CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke); + LocationSummary* locations = invoke->GetLocations(); + if (locations == nullptr) { + return; + } + + HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); + HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); + HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); + + if (src_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(src_pos->GetValue())) { + locations->SetInAt(1, Location::RequiresRegister()); + } + if (dest_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(dest_pos->GetValue())) { + locations->SetInAt(3, Location::RequiresRegister()); + } + if (length != nullptr && !assembler_->ShifterOperandCanAlwaysHold(length->GetValue())) { + locations->SetInAt(4, Location::RequiresRegister()); + } + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Temporary register IP cannot be used in + // ReadBarrierSystemArrayCopySlowPathARM (because that register + // is clobbered by ReadBarrierMarkRegX entry points). Get an extra + // temporary register from the register allocator. + locations->AddTemp(Location::RequiresRegister()); + } +} + +static void CheckPosition(ArmVIXLAssembler* assembler, + Location pos, + vixl32::Register input, + Location length, + SlowPathCodeARMVIXL* slow_path, + vixl32::Register temp, + bool length_is_input_length = false) { + // Where is the length in the Array? + const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value(); + + if (pos.IsConstant()) { + int32_t pos_const = Int32ConstantFrom(pos); + if (pos_const == 0) { + if (!length_is_input_length) { + // Check that length(input) >= length. + __ Ldr(temp, MemOperand(input, length_offset)); + if (length.IsConstant()) { + __ Cmp(temp, Int32ConstantFrom(length)); + } else { + __ Cmp(temp, RegisterFrom(length)); + } + __ B(lt, slow_path->GetEntryLabel()); + } + } else { + // Check that length(input) >= pos. + __ Ldr(temp, MemOperand(input, length_offset)); + __ Subs(temp, temp, pos_const); + __ B(lt, slow_path->GetEntryLabel()); + + // Check that (length(input) - pos) >= length. + if (length.IsConstant()) { + __ Cmp(temp, Int32ConstantFrom(length)); + } else { + __ Cmp(temp, RegisterFrom(length)); + } + __ B(lt, slow_path->GetEntryLabel()); + } + } else if (length_is_input_length) { + // The only way the copy can succeed is if pos is zero. + vixl32::Register pos_reg = RegisterFrom(pos); + __ Cbnz(pos_reg, slow_path->GetEntryLabel()); + } else { + // Check that pos >= 0. + vixl32::Register pos_reg = RegisterFrom(pos); + __ Cmp(pos_reg, 0); + __ B(lt, slow_path->GetEntryLabel()); + + // Check that pos <= length(input). + __ Ldr(temp, MemOperand(input, length_offset)); + __ Subs(temp, temp, pos_reg); + __ B(lt, slow_path->GetEntryLabel()); + + // Check that (length(input) - pos) >= length. + if (length.IsConstant()) { + __ Cmp(temp, Int32ConstantFrom(length)); + } else { + __ Cmp(temp, RegisterFrom(length)); + } + __ B(lt, slow_path->GetEntryLabel()); + } +} + +void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { + // The only read barrier implementation supporting the + // SystemArrayCopy intrinsic is the Baker-style read barriers. + DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); + + ArmVIXLAssembler* assembler = GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); + + vixl32::Register src = InputRegisterAt(invoke, 0); + Location src_pos = locations->InAt(1); + vixl32::Register dest = InputRegisterAt(invoke, 2); + Location dest_pos = locations->InAt(3); + Location length = locations->InAt(4); + Location temp1_loc = locations->GetTemp(0); + vixl32::Register temp1 = RegisterFrom(temp1_loc); + Location temp2_loc = locations->GetTemp(1); + vixl32::Register temp2 = RegisterFrom(temp2_loc); + Location temp3_loc = locations->GetTemp(2); + vixl32::Register temp3 = RegisterFrom(temp3_loc); + + SlowPathCodeARMVIXL* intrinsic_slow_path = new (GetAllocator()) IntrinsicSlowPathARMVIXL(invoke); + codegen_->AddSlowPath(intrinsic_slow_path); + + vixl32::Label conditions_on_positions_validated; + SystemArrayCopyOptimizations optimizations(invoke); + + // If source and destination are the same, we go to slow path if we need to do + // forward copying. + if (src_pos.IsConstant()) { + int32_t src_pos_constant = Int32ConstantFrom(src_pos); + if (dest_pos.IsConstant()) { + int32_t dest_pos_constant = Int32ConstantFrom(dest_pos); + if (optimizations.GetDestinationIsSource()) { + // Checked when building locations. + DCHECK_GE(src_pos_constant, dest_pos_constant); + } else if (src_pos_constant < dest_pos_constant) { + __ Cmp(src, dest); + __ B(eq, intrinsic_slow_path->GetEntryLabel()); + } + + // Checked when building locations. + DCHECK(!optimizations.GetDestinationIsSource() + || (src_pos_constant >= Int32ConstantFrom(dest_pos))); + } else { + if (!optimizations.GetDestinationIsSource()) { + __ Cmp(src, dest); + __ B(ne, &conditions_on_positions_validated); + } + __ Cmp(RegisterFrom(dest_pos), src_pos_constant); + __ B(gt, intrinsic_slow_path->GetEntryLabel()); + } + } else { + if (!optimizations.GetDestinationIsSource()) { + __ Cmp(src, dest); + __ B(ne, &conditions_on_positions_validated); + } + if (dest_pos.IsConstant()) { + int32_t dest_pos_constant = Int32ConstantFrom(dest_pos); + __ Cmp(RegisterFrom(src_pos), dest_pos_constant); + } else { + __ Cmp(RegisterFrom(src_pos), RegisterFrom(dest_pos)); + } + __ B(lt, intrinsic_slow_path->GetEntryLabel()); + } + + __ Bind(&conditions_on_positions_validated); + + if (!optimizations.GetSourceIsNotNull()) { + // Bail out if the source is null. + __ Cbz(src, intrinsic_slow_path->GetEntryLabel()); + } + + if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { + // Bail out if the destination is null. + __ Cbz(dest, intrinsic_slow_path->GetEntryLabel()); + } + + // If the length is negative, bail out. + // We have already checked in the LocationsBuilder for the constant case. + if (!length.IsConstant() && + !optimizations.GetCountIsSourceLength() && + !optimizations.GetCountIsDestinationLength()) { + __ Cmp(RegisterFrom(length), 0); + __ B(lt, intrinsic_slow_path->GetEntryLabel()); + } + + // Validity checks: source. + CheckPosition(assembler, + src_pos, + src, + length, + intrinsic_slow_path, + temp1, + optimizations.GetCountIsSourceLength()); + + // Validity checks: dest. + CheckPosition(assembler, + dest_pos, + dest, + length, + intrinsic_slow_path, + temp1, + optimizations.GetCountIsDestinationLength()); + + if (!optimizations.GetDoesNotNeedTypeCheck()) { + // Check whether all elements of the source array are assignable to the component + // type of the destination array. We do two checks: the classes are the same, + // or the destination is Object[]. If none of these checks succeed, we go to the + // slow path. + + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + if (!optimizations.GetSourceIsNonPrimitiveArray()) { + // /* HeapReference<Class> */ temp1 = src->klass_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp1_loc, src, class_offset, temp2_loc, /* needs_null_check */ false); + // Bail out if the source is not a non primitive array. + // /* HeapReference<Class> */ temp1 = temp1->component_type_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp1_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); + __ Cbz(temp1, intrinsic_slow_path->GetEntryLabel()); + // If heap poisoning is enabled, `temp1` has been unpoisoned + // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. + // /* uint16_t */ temp1 = static_cast<uint16>(temp1->primitive_type_); + __ Ldrh(temp1, MemOperand(temp1, primitive_offset)); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); + } + + // /* HeapReference<Class> */ temp1 = dest->klass_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp1_loc, dest, class_offset, temp2_loc, /* needs_null_check */ false); + + if (!optimizations.GetDestinationIsNonPrimitiveArray()) { + // Bail out if the destination is not a non primitive array. + // + // Register `temp1` is not trashed by the read barrier emitted + // by GenerateFieldLoadWithBakerReadBarrier below, as that + // method produces a call to a ReadBarrierMarkRegX entry point, + // which saves all potentially live registers, including + // temporaries such a `temp1`. + // /* HeapReference<Class> */ temp2 = temp1->component_type_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp2_loc, temp1, component_offset, temp3_loc, /* needs_null_check */ false); + __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel()); + // If heap poisoning is enabled, `temp2` has been unpoisoned + // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. + // /* uint16_t */ temp2 = static_cast<uint16>(temp2->primitive_type_); + __ Ldrh(temp2, MemOperand(temp2, primitive_offset)); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel()); + } + + // For the same reason given earlier, `temp1` is not trashed by the + // read barrier emitted by GenerateFieldLoadWithBakerReadBarrier below. + // /* HeapReference<Class> */ temp2 = src->klass_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp2_loc, src, class_offset, temp3_loc, /* needs_null_check */ false); + // Note: if heap poisoning is on, we are comparing two unpoisoned references here. + __ Cmp(temp1, temp2); + + if (optimizations.GetDestinationIsTypedObjectArray()) { + vixl32::Label do_copy; + __ B(eq, &do_copy); + // /* HeapReference<Class> */ temp1 = temp1->component_type_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp1_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); + // /* HeapReference<Class> */ temp1 = temp1->super_class_ + // We do not need to emit a read barrier for the following + // heap reference load, as `temp1` is only used in a + // comparison with null below, and this reference is not + // kept afterwards. + __ Ldr(temp1, MemOperand(temp1, super_offset)); + __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); + __ Bind(&do_copy); + } else { + __ B(ne, intrinsic_slow_path->GetEntryLabel()); + } + } else { + // Non read barrier code. + + // /* HeapReference<Class> */ temp1 = dest->klass_ + __ Ldr(temp1, MemOperand(dest, class_offset)); + // /* HeapReference<Class> */ temp2 = src->klass_ + __ Ldr(temp2, MemOperand(src, class_offset)); + bool did_unpoison = false; + if (!optimizations.GetDestinationIsNonPrimitiveArray() || + !optimizations.GetSourceIsNonPrimitiveArray()) { + // One or two of the references need to be unpoisoned. Unpoison them + // both to make the identity check valid. + assembler->MaybeUnpoisonHeapReference(temp1); + assembler->MaybeUnpoisonHeapReference(temp2); + did_unpoison = true; + } + + if (!optimizations.GetDestinationIsNonPrimitiveArray()) { + // Bail out if the destination is not a non primitive array. + // /* HeapReference<Class> */ temp3 = temp1->component_type_ + __ Ldr(temp3, MemOperand(temp1, component_offset)); + __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); + assembler->MaybeUnpoisonHeapReference(temp3); + // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); + __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel()); + } + + if (!optimizations.GetSourceIsNonPrimitiveArray()) { + // Bail out if the source is not a non primitive array. + // /* HeapReference<Class> */ temp3 = temp2->component_type_ + __ Ldr(temp3, MemOperand(temp2, component_offset)); + __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); + assembler->MaybeUnpoisonHeapReference(temp3); + // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); + __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel()); + } + + __ Cmp(temp1, temp2); + + if (optimizations.GetDestinationIsTypedObjectArray()) { + vixl32::Label do_copy; + __ B(eq, &do_copy); + if (!did_unpoison) { + assembler->MaybeUnpoisonHeapReference(temp1); + } + // /* HeapReference<Class> */ temp1 = temp1->component_type_ + __ Ldr(temp1, MemOperand(temp1, component_offset)); + assembler->MaybeUnpoisonHeapReference(temp1); + // /* HeapReference<Class> */ temp1 = temp1->super_class_ + __ Ldr(temp1, MemOperand(temp1, super_offset)); + // No need to unpoison the result, we're comparing against null. + __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); + __ Bind(&do_copy); + } else { + __ B(ne, intrinsic_slow_path->GetEntryLabel()); + } + } + } else if (!optimizations.GetSourceIsNonPrimitiveArray()) { + DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); + // Bail out if the source is not a non primitive array. + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // /* HeapReference<Class> */ temp1 = src->klass_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp1_loc, src, class_offset, temp2_loc, /* needs_null_check */ false); + // /* HeapReference<Class> */ temp3 = temp1->component_type_ + codegen_->GenerateFieldLoadWithBakerReadBarrier( + invoke, temp3_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); + __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); + // If heap poisoning is enabled, `temp3` has been unpoisoned + // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. + } else { + // /* HeapReference<Class> */ temp1 = src->klass_ + __ Ldr(temp1, MemOperand(src, class_offset)); + assembler->MaybeUnpoisonHeapReference(temp1); + // /* HeapReference<Class> */ temp3 = temp1->component_type_ + __ Ldr(temp3, MemOperand(temp1, component_offset)); + __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); + assembler->MaybeUnpoisonHeapReference(temp3); + } + // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); + __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel()); + } + + int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot); + uint32_t element_size_shift = Primitive::ComponentSizeShift(Primitive::kPrimNot); + uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value(); + + // Compute the base source address in `temp1`. + if (src_pos.IsConstant()) { + int32_t constant = Int32ConstantFrom(src_pos); + __ Add(temp1, src, element_size * constant + offset); + } else { + __ Add(temp1, src, Operand(RegisterFrom(src_pos), vixl32::LSL, element_size_shift)); + __ Add(temp1, temp1, offset); + } + + // Compute the end source address in `temp3`. + if (length.IsConstant()) { + int32_t constant = Int32ConstantFrom(length); + __ Add(temp3, temp1, element_size * constant); + } else { + __ Add(temp3, temp1, Operand(RegisterFrom(length), vixl32::LSL, element_size_shift)); + } + + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // The base destination address is computed later, as `temp2` is + // used for intermediate computations. + + // SystemArrayCopy implementation for Baker read barriers (see + // also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier): + // + // if (src_ptr != end_ptr) { + // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); + // lfence; // Load fence or artificial data dependency to prevent load-load reordering + // bool is_gray = (rb_state == ReadBarrier::GrayState()); + // if (is_gray) { + // // Slow-path copy. + // do { + // *dest_ptr++ = MaybePoison(ReadBarrier::Mark(MaybeUnpoison(*src_ptr++))); + // } while (src_ptr != end_ptr) + // } else { + // // Fast-path copy. + // do { + // *dest_ptr++ = *src_ptr++; + // } while (src_ptr != end_ptr) + // } + // } + + vixl32::Label loop, done; + + // Don't enter copy loop if `length == 0`. + __ Cmp(temp1, temp3); + __ B(eq, &done); + + // /* int32_t */ monitor = src->monitor_ + __ Ldr(temp2, MemOperand(src, monitor_offset)); + // /* LockWord */ lock_word = LockWord(monitor) + static_assert(sizeof(LockWord) == sizeof(int32_t), + "art::LockWord and int32_t have different sizes."); + + // Introduce a dependency on the lock_word including the rb_state, + // which shall prevent load-load reordering without using + // a memory barrier (which would be more expensive). + // `src` is unchanged by this operation, but its value now depends + // on `temp2`. + __ Add(src, src, Operand(temp2, vixl32::LSR, 32)); + + // Slow path used to copy array when `src` is gray. + SlowPathCodeARMVIXL* read_barrier_slow_path = + new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathARMVIXL(invoke); + codegen_->AddSlowPath(read_barrier_slow_path); + + // Given the numeric representation, it's enough to check the low bit of the + // rb_state. We do that by shifting the bit out of the lock word with LSRS + // which can be a 16-bit instruction unlike the TST immediate. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + __ Lsrs(temp2, temp2, LockWord::kReadBarrierStateShift + 1); + // Carry flag is the last bit shifted out by LSRS. + __ B(cs, read_barrier_slow_path->GetEntryLabel()); + + // Fast-path copy. + + // Compute the base destination address in `temp2`. + if (dest_pos.IsConstant()) { + int32_t constant = Int32ConstantFrom(dest_pos); + __ Add(temp2, dest, element_size * constant + offset); + } else { + __ Add(temp2, dest, Operand(RegisterFrom(dest_pos), vixl32::LSL, element_size_shift)); + __ Add(temp2, temp2, offset); + } + + // Iterate over the arrays and do a raw copy of the objects. We don't need to + // poison/unpoison. + __ Bind(&loop); + + { + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + const vixl32::Register temp_reg = temps.Acquire(); + + __ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex)); + __ Str(temp_reg, MemOperand(temp2, element_size, PostIndex)); + } + + __ Cmp(temp1, temp3); + __ B(ne, &loop); + + __ Bind(read_barrier_slow_path->GetExitLabel()); + __ Bind(&done); + } else { + // Non read barrier code. + + // Compute the base destination address in `temp2`. + if (dest_pos.IsConstant()) { + int32_t constant = Int32ConstantFrom(dest_pos); + __ Add(temp2, dest, element_size * constant + offset); + } else { + __ Add(temp2, dest, Operand(RegisterFrom(dest_pos), vixl32::LSL, element_size_shift)); + __ Add(temp2, temp2, offset); + } + + // Iterate over the arrays and do a raw copy of the objects. We don't need to + // poison/unpoison. + vixl32::Label loop, done; + __ Cmp(temp1, temp3); + __ B(eq, &done); + __ Bind(&loop); + + { + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + const vixl32::Register temp_reg = temps.Acquire(); + + __ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex)); + __ Str(temp_reg, MemOperand(temp2, element_size, PostIndex)); + } + + __ Cmp(temp1, temp3); + __ B(ne, &loop); + __ Bind(&done); + } + + // We only need one card marking on the destination array. + codegen_->MarkGCCard(temp1, temp2, dest, NoReg, /* value_can_be_null */ false); + + __ Bind(intrinsic_slow_path->GetExitLabel()); +} + +static void CreateFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { + // If the graph is debuggable, all callee-saved floating-point registers are blocked by + // the code generator. Furthermore, the register allocator creates fixed live intervals + // for all caller-saved registers because we are doing a function call. As a result, if + // the input and output locations are unallocated, the register allocator runs out of + // registers and fails; however, a debuggable graph is not the common case. + if (invoke->GetBlock()->GetGraph()->IsDebuggable()) { + return; + } + + DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); + DCHECK_EQ(invoke->InputAt(0)->GetType(), Primitive::kPrimDouble); + DCHECK_EQ(invoke->GetType(), Primitive::kPrimDouble); + + LocationSummary* const locations = new (arena) LocationSummary(invoke, + LocationSummary::kCallOnMainOnly, + kIntrinsified); + const InvokeRuntimeCallingConventionARMVIXL calling_convention; + + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + // Native code uses the soft float ABI. + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0))); + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(1))); +} + +static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { + // If the graph is debuggable, all callee-saved floating-point registers are blocked by + // the code generator. Furthermore, the register allocator creates fixed live intervals + // for all caller-saved registers because we are doing a function call. As a result, if + // the input and output locations are unallocated, the register allocator runs out of + // registers and fails; however, a debuggable graph is not the common case. + if (invoke->GetBlock()->GetGraph()->IsDebuggable()) { + return; + } + + DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); + DCHECK_EQ(invoke->InputAt(0)->GetType(), Primitive::kPrimDouble); + DCHECK_EQ(invoke->InputAt(1)->GetType(), Primitive::kPrimDouble); + DCHECK_EQ(invoke->GetType(), Primitive::kPrimDouble); + + LocationSummary* const locations = new (arena) LocationSummary(invoke, + LocationSummary::kCallOnMainOnly, + kIntrinsified); + const InvokeRuntimeCallingConventionARMVIXL calling_convention; + + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + // Native code uses the soft float ABI. + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0))); + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(1))); + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2))); + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(3))); +} + +static void GenFPToFPCall(HInvoke* invoke, + ArmVIXLAssembler* assembler, + CodeGeneratorARMVIXL* codegen, + QuickEntrypointEnum entry) { + LocationSummary* const locations = invoke->GetLocations(); + + DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); + DCHECK(locations->WillCall() && locations->Intrinsified()); + + // Native code uses the soft float ABI. + __ Vmov(RegisterFrom(locations->GetTemp(0)), + RegisterFrom(locations->GetTemp(1)), + InputDRegisterAt(invoke, 0)); + codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); + __ Vmov(OutputDRegister(invoke), + RegisterFrom(locations->GetTemp(0)), + RegisterFrom(locations->GetTemp(1))); +} + +static void GenFPFPToFPCall(HInvoke* invoke, + ArmVIXLAssembler* assembler, + CodeGeneratorARMVIXL* codegen, + QuickEntrypointEnum entry) { + LocationSummary* const locations = invoke->GetLocations(); + + DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); + DCHECK(locations->WillCall() && locations->Intrinsified()); + + // Native code uses the soft float ABI. + __ Vmov(RegisterFrom(locations->GetTemp(0)), + RegisterFrom(locations->GetTemp(1)), + InputDRegisterAt(invoke, 0)); + __ Vmov(RegisterFrom(locations->GetTemp(2)), + RegisterFrom(locations->GetTemp(3)), + InputDRegisterAt(invoke, 1)); + codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); + __ Vmov(OutputDRegister(invoke), + RegisterFrom(locations->GetTemp(0)), + RegisterFrom(locations->GetTemp(1))); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathCos(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathCos(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCos); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathSin(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathSin(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickSin); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAcos(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAcos(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAcos); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAsin(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAsin(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAsin); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAtan(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAtan(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAtan); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathCbrt(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathCbrt(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCbrt); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathCosh(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathCosh(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCosh); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathExp(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathExp(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickExp); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathExpm1(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathExpm1(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickExpm1); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathLog(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathLog(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickLog); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathLog10(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathLog10(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickLog10); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathSinh(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathSinh(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickSinh); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathTan(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathTan(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickTan); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathTanh(HInvoke* invoke) { + CreateFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathTanh(HInvoke* invoke) { + GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickTanh); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathAtan2(HInvoke* invoke) { + CreateFPFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathAtan2(HInvoke* invoke) { + GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAtan2); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathHypot(HInvoke* invoke) { + CreateFPFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathHypot(HInvoke* invoke) { + GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickHypot); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathNextAfter(HInvoke* invoke) { + CreateFPFPToFPCallLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathNextAfter(HInvoke* invoke) { + GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickNextAfter); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitIntegerReverse(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitIntegerReverse(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Rbit(OutputRegister(invoke), InputRegisterAt(invoke, 0)); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitLongReverse(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitLongReverse(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + vixl32::Register in_reg_lo = LowRegisterFrom(locations->InAt(0)); + vixl32::Register in_reg_hi = HighRegisterFrom(locations->InAt(0)); + vixl32::Register out_reg_lo = LowRegisterFrom(locations->Out()); + vixl32::Register out_reg_hi = HighRegisterFrom(locations->Out()); + + __ Rbit(out_reg_lo, in_reg_hi); + __ Rbit(out_reg_hi, in_reg_lo); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitIntegerReverseBytes(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitIntegerReverseBytes(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Rev(OutputRegister(invoke), InputRegisterAt(invoke, 0)); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitLongReverseBytes(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitLongReverseBytes(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + vixl32::Register in_reg_lo = LowRegisterFrom(locations->InAt(0)); + vixl32::Register in_reg_hi = HighRegisterFrom(locations->InAt(0)); + vixl32::Register out_reg_lo = LowRegisterFrom(locations->Out()); + vixl32::Register out_reg_hi = HighRegisterFrom(locations->Out()); + + __ Rev(out_reg_lo, in_reg_hi); + __ Rev(out_reg_hi, in_reg_lo); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitShortReverseBytes(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitShortReverseBytes(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + __ Revsh(OutputRegister(invoke), InputRegisterAt(invoke, 0)); +} + +static void GenBitCount(HInvoke* instr, Primitive::Type type, ArmVIXLAssembler* assembler) { + DCHECK(Primitive::IsIntOrLongType(type)) << type; + DCHECK_EQ(instr->GetType(), Primitive::kPrimInt); + DCHECK_EQ(Primitive::PrimitiveKind(instr->InputAt(0)->GetType()), type); + + bool is_long = type == Primitive::kPrimLong; + LocationSummary* locations = instr->GetLocations(); + Location in = locations->InAt(0); + vixl32::Register src_0 = is_long ? LowRegisterFrom(in) : RegisterFrom(in); + vixl32::Register src_1 = is_long ? HighRegisterFrom(in) : src_0; + vixl32::SRegister tmp_s = LowSRegisterFrom(locations->GetTemp(0)); + vixl32::DRegister tmp_d = DRegisterFrom(locations->GetTemp(0)); + vixl32::Register out_r = OutputRegister(instr); + + // Move data from core register(s) to temp D-reg for bit count calculation, then move back. + // According to Cortex A57 and A72 optimization guides, compared to transferring to full D-reg, + // transferring data from core reg to upper or lower half of vfp D-reg requires extra latency, + // That's why for integer bit count, we use 'vmov d0, r0, r0' instead of 'vmov d0[0], r0'. + __ Vmov(tmp_d, src_1, src_0); // Temp DReg |--src_1|--src_0| + __ Vcnt(Untyped8, tmp_d, tmp_d); // Temp DReg |c|c|c|c|c|c|c|c| + __ Vpaddl(U8, tmp_d, tmp_d); // Temp DReg |--c|--c|--c|--c| + __ Vpaddl(U16, tmp_d, tmp_d); // Temp DReg |------c|------c| + if (is_long) { + __ Vpaddl(U32, tmp_d, tmp_d); // Temp DReg |--------------c| + } + __ Vmov(out_r, tmp_s); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitIntegerBitCount(HInvoke* invoke) { + CreateIntToIntLocations(arena_, invoke); + invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitIntegerBitCount(HInvoke* invoke) { + GenBitCount(invoke, Primitive::kPrimInt, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitLongBitCount(HInvoke* invoke) { + VisitIntegerBitCount(invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitLongBitCount(HInvoke* invoke) { + GenBitCount(invoke, Primitive::kPrimLong, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetInAt(3, Location::RequiresRegister()); + locations->SetInAt(4, Location::RequiresRegister()); + + // Temporary registers to store lengths of strings and for calculations. + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresRegister()); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) { + ArmVIXLAssembler* assembler = GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + // Check assumption that sizeof(Char) is 2 (used in scaling below). + const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); + DCHECK_EQ(char_size, 2u); + + // Location of data in char array buffer. + const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value(); + + // Location of char array data in string. + const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); + + // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin); + // Since getChars() calls getCharsNoCheck() - we use registers rather than constants. + vixl32::Register srcObj = InputRegisterAt(invoke, 0); + vixl32::Register srcBegin = InputRegisterAt(invoke, 1); + vixl32::Register srcEnd = InputRegisterAt(invoke, 2); + vixl32::Register dstObj = InputRegisterAt(invoke, 3); + vixl32::Register dstBegin = InputRegisterAt(invoke, 4); + + vixl32::Register num_chr = RegisterFrom(locations->GetTemp(0)); + vixl32::Register src_ptr = RegisterFrom(locations->GetTemp(1)); + vixl32::Register dst_ptr = RegisterFrom(locations->GetTemp(2)); + + vixl32::Label done, compressed_string_loop; + // dst to be copied. + __ Add(dst_ptr, dstObj, data_offset); + __ Add(dst_ptr, dst_ptr, Operand(dstBegin, vixl32::LSL, 1)); + + __ Subs(num_chr, srcEnd, srcBegin); + // Early out for valid zero-length retrievals. + __ B(eq, &done); + + // src range to copy. + __ Add(src_ptr, srcObj, value_offset); + + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + vixl32::Register temp; + vixl32::Label compressed_string_preloop; + if (mirror::kUseStringCompression) { + // Location of count in string. + const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); + temp = temps.Acquire(); + // String's length. + __ Ldr(temp, MemOperand(srcObj, count_offset)); + __ Cmp(temp, 0); + temps.Release(temp); + __ B(lt, &compressed_string_preloop); + } + __ Add(src_ptr, src_ptr, Operand(srcBegin, vixl32::LSL, 1)); + + // Do the copy. + vixl32::Label loop, remainder; + + temp = temps.Acquire(); + // Save repairing the value of num_chr on the < 4 character path. + __ Subs(temp, num_chr, 4); + __ B(lt, &remainder); + + // Keep the result of the earlier subs, we are going to fetch at least 4 characters. + __ Mov(num_chr, temp); + + // Main loop used for longer fetches loads and stores 4x16-bit characters at a time. + // (LDRD/STRD fault on unaligned addresses and it's not worth inlining extra code + // to rectify these everywhere this intrinsic applies.) + __ Bind(&loop); + __ Ldr(temp, MemOperand(src_ptr, char_size * 2)); + __ Subs(num_chr, num_chr, 4); + __ Str(temp, MemOperand(dst_ptr, char_size * 2)); + __ Ldr(temp, MemOperand(src_ptr, char_size * 4, PostIndex)); + __ Str(temp, MemOperand(dst_ptr, char_size * 4, PostIndex)); + temps.Release(temp); + __ B(ge, &loop); + + __ Adds(num_chr, num_chr, 4); + __ B(eq, &done); + + // Main loop for < 4 character case and remainder handling. Loads and stores one + // 16-bit Java character at a time. + __ Bind(&remainder); + temp = temps.Acquire(); + __ Ldrh(temp, MemOperand(src_ptr, char_size, PostIndex)); + __ Subs(num_chr, num_chr, 1); + __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); + temps.Release(temp); + __ B(gt, &remainder); + __ B(&done); + + if (mirror::kUseStringCompression) { + const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte); + DCHECK_EQ(c_char_size, 1u); + // Copy loop for compressed src, copying 1 character (8-bit) to (16-bit) at a time. + __ Bind(&compressed_string_preloop); + __ Add(src_ptr, src_ptr, srcBegin); + __ Bind(&compressed_string_loop); + temp = temps.Acquire(); + __ Ldrb(temp, MemOperand(src_ptr, c_char_size, PostIndex)); + __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); + temps.Release(temp); + __ Subs(num_chr, num_chr, 1); + __ B(gt, &compressed_string_loop); + } + + __ Bind(&done); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitFloatIsInfinite(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitFloatIsInfinite(HInvoke* invoke) { + ArmVIXLAssembler* const assembler = GetAssembler(); + const vixl32::Register out = OutputRegister(invoke); + // Shifting left by 1 bit makes the value encodable as an immediate operand; + // we don't care about the sign bit anyway. + constexpr uint32_t infinity = kPositiveInfinityFloat << 1U; + + __ Vmov(out, InputSRegisterAt(invoke, 0)); + // We don't care about the sign bit, so shift left. + __ Lsl(out, out, 1); + __ Eor(out, out, infinity); + // If the result is 0, then it has 32 leading zeros, and less than that otherwise. + __ Clz(out, out); + // Any number less than 32 logically shifted right by 5 bits results in 0; + // the same operation on 32 yields 1. + __ Lsr(out, out, 5); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitDoubleIsInfinite(HInvoke* invoke) { + CreateFPToIntLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitDoubleIsInfinite(HInvoke* invoke) { + ArmVIXLAssembler* const assembler = GetAssembler(); + const vixl32::Register out = OutputRegister(invoke); + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + const vixl32::Register temp = temps.Acquire(); + // The highest 32 bits of double precision positive infinity separated into + // two constants encodable as immediate operands. + constexpr uint32_t infinity_high = 0x7f000000U; + constexpr uint32_t infinity_high2 = 0x00f00000U; + + static_assert((infinity_high | infinity_high2) == + static_cast<uint32_t>(kPositiveInfinityDouble >> 32U), + "The constants do not add up to the high 32 bits of double " + "precision positive infinity."); + __ Vmov(temp, out, InputDRegisterAt(invoke, 0)); + __ Eor(out, out, infinity_high); + __ Eor(out, out, infinity_high2); + // We don't care about the sign bit, so shift left. + __ Orr(out, temp, Operand(out, vixl32::LSL, 1)); + // If the result is 0, then it has 32 leading zeros, and less than that otherwise. + __ Clz(out, out); + // Any number less than 32 logically shifted right by 5 bits results in 0; + // the same operation on 32 yields 1. + __ Lsr(out, out, 5); +} + +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMinDoubleDouble) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMinFloatFloat) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMaxDoubleDouble) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMaxFloatFloat) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMinLongLong) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMaxLongLong) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathCeil) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathFloor) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathRint) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathRoundDouble) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathRoundFloat) // Could be done by changing rounding mode, maybe? +UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeCASLong) // High register pressure. +UNIMPLEMENTED_INTRINSIC(ARMVIXL, SystemArrayCopyChar) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, ReferenceGetReferent) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, IntegerHighestOneBit) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, LongHighestOneBit) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, IntegerLowestOneBit) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, LongLowestOneBit) + +// 1.8. +UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeGetAndAddInt) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeGetAndAddLong) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeGetAndSetInt) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeGetAndSetLong) +UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeGetAndSetObject) + +UNREACHABLE_INTRINSICS(ARMVIXL) + +#undef __ + +} // namespace arm +} // namespace art diff --git a/compiler/optimizing/intrinsics_arm_vixl.h b/compiler/optimizing/intrinsics_arm_vixl.h new file mode 100644 index 0000000000..6e79cb76a1 --- /dev/null +++ b/compiler/optimizing/intrinsics_arm_vixl.h @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_VIXL_H_ +#define ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_VIXL_H_ + +#include "intrinsics.h" +#include "utils/arm/assembler_arm_vixl.h" + +namespace art { + +namespace arm { + +class ArmVIXLAssembler; +class CodeGeneratorARMVIXL; + +class IntrinsicLocationsBuilderARMVIXL FINAL : public IntrinsicVisitor { + public: + explicit IntrinsicLocationsBuilderARMVIXL(CodeGeneratorARMVIXL* codegen); + + // Define visitor methods. + +#define OPTIMIZING_INTRINSICS(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \ + void Visit ## Name(HInvoke* invoke) OVERRIDE; +#include "intrinsics_list.h" +INTRINSICS_LIST(OPTIMIZING_INTRINSICS) +#undef INTRINSICS_LIST +#undef OPTIMIZING_INTRINSICS + + // Check whether an invoke is an intrinsic, and if so, create a location summary. Returns whether + // a corresponding LocationSummary with the intrinsified_ flag set was generated and attached to + // the invoke. + bool TryDispatch(HInvoke* invoke); + + private: + ArenaAllocator* arena_; + ArmVIXLAssembler* assembler_; + const ArmInstructionSetFeatures& features_; + + DISALLOW_COPY_AND_ASSIGN(IntrinsicLocationsBuilderARMVIXL); +}; + +class IntrinsicCodeGeneratorARMVIXL FINAL : public IntrinsicVisitor { + public: + explicit IntrinsicCodeGeneratorARMVIXL(CodeGeneratorARMVIXL* codegen) : codegen_(codegen) {} + + // Define visitor methods. + +#define OPTIMIZING_INTRINSICS(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \ + void Visit ## Name(HInvoke* invoke) OVERRIDE; +#include "intrinsics_list.h" +INTRINSICS_LIST(OPTIMIZING_INTRINSICS) +#undef INTRINSICS_LIST +#undef OPTIMIZING_INTRINSICS + + private: + ArenaAllocator* GetAllocator(); + ArmVIXLAssembler* GetAssembler(); + + CodeGeneratorARMVIXL* codegen_; + + DISALLOW_COPY_AND_ASSIGN(IntrinsicCodeGeneratorARMVIXL); +}; + +} // namespace arm +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_VIXL_H_ diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index aae3899847..43682c5633 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -3200,7 +3200,7 @@ void IntrinsicCodeGeneratorX86::VisitSystemArrayCopy(HInvoke* invoke) { // if (src_ptr != end_ptr) { // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // // Slow-path copy. // for (size_t i = 0; i != length; ++i) { @@ -3222,14 +3222,13 @@ void IntrinsicCodeGeneratorX86::VisitSystemArrayCopy(HInvoke* invoke) { __ j(kEqual, &done); // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte; constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte; constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position); - // if (rb_state == ReadBarrier::gray_ptr_) + // if (rb_state == ReadBarrier::GrayState()) // goto slow_path; // At this point, just do the "if" and make sure that flags are preserved until the branch. __ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value)); diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index cdef22f6de..de2606c327 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -1399,7 +1399,7 @@ void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) { // if (src_ptr != end_ptr) { // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // // Slow-path copy. // do { @@ -1420,14 +1420,13 @@ void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) { __ j(kEqual, &done); // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1"); - static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2"); + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte; constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte; constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position); - // if (rb_state == ReadBarrier::gray_ptr_) + // if (rb_state == ReadBarrier::GrayState()) // goto slow_path; // At this point, just do the "if" and make sure that flags are preserved until the branch. __ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value)); diff --git a/compiler/optimizing/load_store_elimination.cc b/compiler/optimizing/load_store_elimination.cc index b91e9e6868..5b2cbf783d 100644 --- a/compiler/optimizing/load_store_elimination.cc +++ b/compiler/optimizing/load_store_elimination.cc @@ -663,27 +663,59 @@ class LSEVisitor : public HGraphVisitor { if (predecessors.size() == 0) { return; } + ArenaVector<HInstruction*>& heap_values = heap_values_for_[block->GetBlockId()]; for (size_t i = 0; i < heap_values.size(); i++) { - HInstruction* pred0_value = heap_values_for_[predecessors[0]->GetBlockId()][i]; - heap_values[i] = pred0_value; - if (pred0_value != kUnknownHeapValue) { - for (size_t j = 1; j < predecessors.size(); j++) { - HInstruction* pred_value = heap_values_for_[predecessors[j]->GetBlockId()][i]; - if (pred_value != pred0_value) { - heap_values[i] = kUnknownHeapValue; - break; - } + HInstruction* merged_value = nullptr; + // Whether merged_value is a result that's merged from all predecessors. + bool from_all_predecessors = true; + ReferenceInfo* ref_info = heap_location_collector_.GetHeapLocation(i)->GetReferenceInfo(); + HInstruction* singleton_ref = nullptr; + if (ref_info->IsSingletonAndNotReturned()) { + // We do more analysis of liveness when merging heap values for such + // cases since stores into such references may potentially be eliminated. + singleton_ref = ref_info->GetReference(); + } + + for (HBasicBlock* predecessor : predecessors) { + HInstruction* pred_value = heap_values_for_[predecessor->GetBlockId()][i]; + if ((singleton_ref != nullptr) && + !singleton_ref->GetBlock()->Dominates(predecessor)) { + // singleton_ref is not live in this predecessor. Skip this predecessor since + // it does not really have the location. + DCHECK_EQ(pred_value, kUnknownHeapValue); + from_all_predecessors = false; + continue; + } + if (merged_value == nullptr) { + // First seen heap value. + merged_value = pred_value; + } else if (pred_value != merged_value) { + // There are conflicting values. + merged_value = kUnknownHeapValue; + break; } } - if (heap_values[i] == kUnknownHeapValue) { + if (merged_value == kUnknownHeapValue) { + // There are conflicting heap values from different predecessors. // Keep the last store in each predecessor since future loads cannot be eliminated. - for (size_t j = 0; j < predecessors.size(); j++) { - ArenaVector<HInstruction*>& pred_values = heap_values_for_[predecessors[j]->GetBlockId()]; + for (HBasicBlock* predecessor : predecessors) { + ArenaVector<HInstruction*>& pred_values = heap_values_for_[predecessor->GetBlockId()]; KeepIfIsStore(pred_values[i]); } } + + if ((merged_value == nullptr) || !from_all_predecessors) { + DCHECK(singleton_ref != nullptr); + DCHECK((singleton_ref->GetBlock() == block) || + !singleton_ref->GetBlock()->Dominates(block)); + // singleton_ref is not defined before block or defined only in some of its + // predecessors, so block doesn't really have the location at its entry. + heap_values[i] = kUnknownHeapValue; + } else { + heap_values[i] = merged_value; + } } } diff --git a/compiler/optimizing/locations.cc b/compiler/optimizing/locations.cc index d157509758..a9fe209063 100644 --- a/compiler/optimizing/locations.cc +++ b/compiler/optimizing/locations.cc @@ -16,11 +16,16 @@ #include "locations.h" +#include <type_traits> + #include "nodes.h" #include "code_generator.h" namespace art { +// Verify that Location is trivially copyable. +static_assert(std::is_trivially_copyable<Location>::value, "Location should be trivially copyable"); + LocationSummary::LocationSummary(HInstruction* instruction, CallKind call_kind, bool intrinsified) diff --git a/compiler/optimizing/locations.h b/compiler/optimizing/locations.h index da27928ef2..52747c0cc4 100644 --- a/compiler/optimizing/locations.h +++ b/compiler/optimizing/locations.h @@ -91,12 +91,9 @@ class Location : public ValueObject { DCHECK(!IsValid()); } - Location(const Location& other) : value_(other.value_) {} + Location(const Location& other) = default; - Location& operator=(const Location& other) { - value_ = other.value_; - return *this; - } + Location& operator=(const Location& other) = default; bool IsConstant() const { return (value_ & kLocationConstantMask) == kConstant; @@ -328,7 +325,6 @@ class Location : public ValueObject { LOG(FATAL) << "Should not use this location kind"; } UNREACHABLE(); - return "?"; } // Unallocated locations. diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index 51be1d1e91..f4616e39e6 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -28,6 +28,30 @@ static void RemoveFromCycle(HInstruction* instruction) { instruction->GetBlock()->RemoveInstructionOrPhi(instruction, /*ensure_safety=*/ false); } +// Detect a goto block and sets succ to the single successor. +static bool IsGotoBlock(HBasicBlock* block, /*out*/ HBasicBlock** succ) { + if (block->GetPredecessors().size() == 1 && + block->GetSuccessors().size() == 1 && + block->IsSingleGoto()) { + *succ = block->GetSingleSuccessor(); + return true; + } + return false; +} + +// Detect an early exit loop. +static bool IsEarlyExit(HLoopInformation* loop_info) { + HBlocksInLoopReversePostOrderIterator it_loop(*loop_info); + for (it_loop.Advance(); !it_loop.Done(); it_loop.Advance()) { + for (HBasicBlock* successor : it_loop.Current()->GetSuccessors()) { + if (!loop_info->Contains(*successor)) { + return true; + } + } + } + return false; +} + // // Class methods. // @@ -168,7 +192,9 @@ void HLoopOptimization::SimplifyInduction(LoopNode* node) { int32_t use_count = 0; if (IsPhiInduction(phi) && IsOnlyUsedAfterLoop(node->loop_info, phi, &use_count) && - TryReplaceWithLastValue(phi, use_count, preheader)) { + // No uses, or no early-exit with proper replacement. + (use_count == 0 || + (!IsEarlyExit(node->loop_info) && TryReplaceWithLastValue(phi, preheader)))) { for (HInstruction* i : *iset_) { RemoveFromCycle(i); } @@ -178,31 +204,57 @@ void HLoopOptimization::SimplifyInduction(LoopNode* node) { } void HLoopOptimization::SimplifyBlocks(LoopNode* node) { - for (HBlocksInLoopIterator it(*node->loop_info); !it.Done(); it.Advance()) { - HBasicBlock* block = it.Current(); - // Remove instructions that are dead. - for (HBackwardInstructionIterator i(block->GetInstructions()); !i.Done(); i.Advance()) { - HInstruction* instruction = i.Current(); - if (instruction->IsDeadAndRemovable()) { - block->RemoveInstruction(instruction); + // Repeat the block simplifications until no more changes occur. Note that since + // each simplification consists of eliminating code (without introducing new code), + // this process is always finite. + bool changed; + do { + changed = false; + // Iterate over all basic blocks in the loop-body. + for (HBlocksInLoopIterator it(*node->loop_info); !it.Done(); it.Advance()) { + HBasicBlock* block = it.Current(); + // Remove dead instructions from the loop-body. + for (HBackwardInstructionIterator i(block->GetInstructions()); !i.Done(); i.Advance()) { + HInstruction* instruction = i.Current(); + if (instruction->IsDeadAndRemovable()) { + changed = true; + block->RemoveInstruction(instruction); + } } - } - // Remove trivial control flow blocks from the loop-body. - if (block->GetPredecessors().size() == 1 && - block->GetSuccessors().size() == 1 && - block->GetFirstInstruction()->IsGoto()) { - HBasicBlock* pred = block->GetSinglePredecessor(); - HBasicBlock* succ = block->GetSingleSuccessor(); - if (succ->GetPredecessors().size() == 1) { + // Remove trivial control flow blocks from the loop-body. + HBasicBlock* succ = nullptr; + if (IsGotoBlock(block, &succ) && succ->GetPredecessors().size() == 1) { + // Trivial goto block can be removed. + HBasicBlock* pred = block->GetSinglePredecessor(); + changed = true; pred->ReplaceSuccessor(block, succ); - block->ClearDominanceInformation(); - block->SetDominator(pred); // needed by next disconnect. + block->RemoveDominatedBlock(succ); block->DisconnectAndDelete(); pred->AddDominatedBlock(succ); succ->SetDominator(pred); + } else if (block->GetSuccessors().size() == 2) { + // Trivial if block can be bypassed to either branch. + HBasicBlock* succ0 = block->GetSuccessors()[0]; + HBasicBlock* succ1 = block->GetSuccessors()[1]; + HBasicBlock* meet0 = nullptr; + HBasicBlock* meet1 = nullptr; + if (succ0 != succ1 && + IsGotoBlock(succ0, &meet0) && + IsGotoBlock(succ1, &meet1) && + meet0 == meet1 && // meets again + meet0 != block && // no self-loop + meet0->GetPhis().IsEmpty()) { // not used for merging + changed = true; + succ0->DisconnectAndDelete(); + if (block->Dominates(meet0)) { + block->RemoveDominatedBlock(meet0); + succ1->AddDominatedBlock(meet0); + meet0->SetDominator(succ1); + } + } } } - } + } while (changed); } void HLoopOptimization::RemoveIfEmptyInnerLoop(LoopNode* node) { @@ -240,12 +292,12 @@ void HLoopOptimization::RemoveIfEmptyInnerLoop(LoopNode* node) { if (IsEmptyHeader(header) && IsEmptyBody(body) && IsOnlyUsedAfterLoop(node->loop_info, header->GetFirstPhi(), &use_count) && - TryReplaceWithLastValue(header->GetFirstPhi(), use_count, preheader)) { + // No uses, or proper replacement. + (use_count == 0 || TryReplaceWithLastValue(header->GetFirstPhi(), preheader))) { body->DisconnectAndDelete(); exit->RemovePredecessor(header); header->RemoveSuccessor(exit); - header->ClearDominanceInformation(); - header->SetDominator(preheader); // needed by next disconnect. + header->RemoveDominatedBlock(exit); header->DisconnectAndDelete(); preheader->AddSuccessor(exit); preheader->AddInstruction(new (graph_->GetArena()) HGoto()); // global allocator @@ -259,22 +311,23 @@ void HLoopOptimization::RemoveIfEmptyInnerLoop(LoopNode* node) { bool HLoopOptimization::IsPhiInduction(HPhi* phi) { ArenaSet<HInstruction*>* set = induction_range_.LookupCycle(phi); if (set != nullptr) { + DCHECK(iset_->empty()); for (HInstruction* i : *set) { - // Check that, other than phi, instruction are removable with uses contained in the cycle. - // TODO: investigate what cases are no longer in the graph. - if (i != phi) { - if (!i->IsInBlock() || !i->IsRemovable()) { - return false; - } + // Check that, other than instructions that are no longer in the graph (removed earlier) + // each instruction is removable and, other than the phi, uses are contained in the cycle. + if (!i->IsInBlock()) { + continue; + } else if (!i->IsRemovable()) { + return false; + } else if (i != phi) { for (const HUseListNode<HInstruction*>& use : i->GetUses()) { if (set->find(use.GetUser()) == set->end()) { return false; } } } + iset_->insert(i); // copy } - DCHECK(iset_->empty()); - iset_->insert(set->begin(), set->end()); // copy return true; } return false; @@ -358,20 +411,16 @@ void HLoopOptimization::ReplaceAllUses(HInstruction* instruction, HInstruction* } } -bool HLoopOptimization::TryReplaceWithLastValue(HInstruction* instruction, - int32_t use_count, - HBasicBlock* block) { - // If true uses appear after the loop, replace these uses with the last value. Environment - // uses can consume this value too, since any first true use is outside the loop (although - // this may imply that de-opting may look "ahead" a bit on the phi value). If there are only - // environment uses, the value is dropped altogether, since the computations have no effect. - if (use_count > 0) { - if (!induction_range_.CanGenerateLastValue(instruction)) { - return false; - } +bool HLoopOptimization::TryReplaceWithLastValue(HInstruction* instruction, HBasicBlock* block) { + // Try to replace outside uses with the last value. Environment uses can consume this + // value too, since any first true use is outside the loop (although this may imply + // that de-opting may look "ahead" a bit on the phi value). If there are only environment + // uses, the value is dropped altogether, since the computations have no effect. + if (induction_range_.CanGenerateLastValue(instruction)) { ReplaceAllUses(instruction, induction_range_.GenerateLastValue(instruction, graph_, block)); + return true; } - return true; + return false; } } // namespace art diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h index e18d17531e..3391bef4e9 100644 --- a/compiler/optimizing/loop_optimization.h +++ b/compiler/optimizing/loop_optimization.h @@ -72,9 +72,7 @@ class HLoopOptimization : public HOptimization { HInstruction* instruction, /*out*/ int32_t* use_count); void ReplaceAllUses(HInstruction* instruction, HInstruction* replacement); - bool TryReplaceWithLastValue(HInstruction* instruction, - int32_t use_count, - HBasicBlock* block); + bool TryReplaceWithLastValue(HInstruction* instruction, HBasicBlock* block); // Range information based on prior induction variable analysis. InductionVarRange induction_range_; diff --git a/compiler/optimizing/register_allocation_resolver.cc b/compiler/optimizing/register_allocation_resolver.cc index caf66474eb..5991791a15 100644 --- a/compiler/optimizing/register_allocation_resolver.cc +++ b/compiler/optimizing/register_allocation_resolver.cc @@ -374,7 +374,9 @@ void RegisterAllocationResolver::ConnectSiblings(LiveInterval* interval) { if (current->GetType() == Primitive::kPrimNot) { DCHECK(interval->GetDefinedBy()->IsActualObject()) << interval->GetDefinedBy()->DebugName() - << "@" << safepoint_position->GetInstruction()->DebugName(); + << '(' << interval->GetDefinedBy()->GetId() << ')' + << "@" << safepoint_position->GetInstruction()->DebugName() + << '(' << safepoint_position->GetInstruction()->GetId() << ')'; LocationSummary* locations = safepoint_position->GetLocations(); if (current->GetParent()->HasSpillSlot()) { locations->SetStackBit(current->GetParent()->GetSpillSlot() / kVRegSize); diff --git a/compiler/utils/assembler_test.h b/compiler/utils/assembler_test.h index 9c65280407..b34e125866 100644 --- a/compiler/utils/assembler_test.h +++ b/compiler/utils/assembler_test.h @@ -51,30 +51,30 @@ class AssemblerTest : public testing::Test { typedef std::string (*TestFn)(AssemblerTest* assembler_test, Ass* assembler); - void DriverFn(TestFn f, std::string test_name) { + void DriverFn(TestFn f, const std::string& test_name) { DriverWrapper(f(this, assembler_.get()), test_name); } // This driver assumes the assembler has already been called. - void DriverStr(std::string assembly_string, std::string test_name) { + void DriverStr(const std::string& assembly_string, const std::string& test_name) { DriverWrapper(assembly_string, test_name); } - std::string RepeatR(void (Ass::*f)(Reg), std::string fmt) { + std::string RepeatR(void (Ass::*f)(Reg), const std::string& fmt) { return RepeatTemplatedRegister<Reg>(f, GetRegisters(), &AssemblerTest::GetRegName<RegisterView::kUsePrimaryName>, fmt); } - std::string Repeatr(void (Ass::*f)(Reg), std::string fmt) { + std::string Repeatr(void (Ass::*f)(Reg), const std::string& fmt) { return RepeatTemplatedRegister<Reg>(f, GetRegisters(), &AssemblerTest::GetRegName<RegisterView::kUseSecondaryName>, fmt); } - std::string RepeatRR(void (Ass::*f)(Reg, Reg), std::string fmt) { + std::string RepeatRR(void (Ass::*f)(Reg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<Reg, Reg>(f, GetRegisters(), GetRegisters(), @@ -83,7 +83,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatRRNoDupes(void (Ass::*f)(Reg, Reg), std::string fmt) { + std::string RepeatRRNoDupes(void (Ass::*f)(Reg, Reg), const std::string& fmt) { return RepeatTemplatedRegistersNoDupes<Reg, Reg>(f, GetRegisters(), GetRegisters(), @@ -92,7 +92,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string Repeatrr(void (Ass::*f)(Reg, Reg), std::string fmt) { + std::string Repeatrr(void (Ass::*f)(Reg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<Reg, Reg>(f, GetRegisters(), GetRegisters(), @@ -101,7 +101,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatRRR(void (Ass::*f)(Reg, Reg, Reg), std::string fmt) { + std::string RepeatRRR(void (Ass::*f)(Reg, Reg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<Reg, Reg, Reg>(f, GetRegisters(), GetRegisters(), @@ -112,7 +112,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string Repeatrb(void (Ass::*f)(Reg, Reg), std::string fmt) { + std::string Repeatrb(void (Ass::*f)(Reg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<Reg, Reg>(f, GetRegisters(), GetRegisters(), @@ -121,7 +121,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatRr(void (Ass::*f)(Reg, Reg), std::string fmt) { + std::string RepeatRr(void (Ass::*f)(Reg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<Reg, Reg>(f, GetRegisters(), GetRegisters(), @@ -130,11 +130,11 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatRI(void (Ass::*f)(Reg, const Imm&), size_t imm_bytes, std::string fmt) { + std::string RepeatRI(void (Ass::*f)(Reg, const Imm&), size_t imm_bytes, const std::string& fmt) { return RepeatRegisterImm<RegisterView::kUsePrimaryName>(f, imm_bytes, fmt); } - std::string Repeatri(void (Ass::*f)(Reg, const Imm&), size_t imm_bytes, std::string fmt) { + std::string Repeatri(void (Ass::*f)(Reg, const Imm&), size_t imm_bytes, const std::string& fmt) { return RepeatRegisterImm<RegisterView::kUseSecondaryName>(f, imm_bytes, fmt); } @@ -145,7 +145,7 @@ class AssemblerTest : public testing::Test { const std::vector<Reg2*> reg2_registers, std::string (AssemblerTest::*GetName1)(const Reg1&), std::string (AssemblerTest::*GetName2)(const Reg2&), - std::string fmt) { + const std::string& fmt) { std::string str; std::vector<int64_t> imms = CreateImmediateValuesBits(abs(imm_bits), (imm_bits > 0)); @@ -195,7 +195,7 @@ class AssemblerTest : public testing::Test { std::string (AssemblerTest::*GetName1)(const Reg1&), std::string (AssemblerTest::*GetName2)(const Reg2&), int imm_bits, - std::string fmt) { + const std::string& fmt) { std::vector<int64_t> imms = CreateImmediateValuesBits(abs(imm_bits), (imm_bits > 0)); WarnOnCombinations(reg1_registers.size() * reg2_registers.size() * imms.size()); @@ -245,7 +245,7 @@ class AssemblerTest : public testing::Test { int imm_bits, const std::vector<Reg*> registers, std::string (AssemblerTest::*GetName)(const RegType&), - std::string fmt) { + const std::string& fmt) { std::string str; std::vector<int64_t> imms = CreateImmediateValuesBits(abs(imm_bits), (imm_bits > 0)); @@ -281,7 +281,7 @@ class AssemblerTest : public testing::Test { } template <typename ImmType> - std::string RepeatRRIb(void (Ass::*f)(Reg, Reg, ImmType), int imm_bits, std::string fmt) { + std::string RepeatRRIb(void (Ass::*f)(Reg, Reg, ImmType), int imm_bits, const std::string& fmt) { return RepeatTemplatedRegistersImmBits<Reg, Reg, ImmType>(f, imm_bits, GetRegisters(), @@ -292,7 +292,7 @@ class AssemblerTest : public testing::Test { } template <typename ImmType> - std::string RepeatRIb(void (Ass::*f)(Reg, ImmType), int imm_bits, std::string fmt) { + std::string RepeatRIb(void (Ass::*f)(Reg, ImmType), int imm_bits, const std::string& fmt) { return RepeatTemplatedRegisterImmBits<Reg, ImmType>(f, imm_bits, GetRegisters(), @@ -301,7 +301,9 @@ class AssemblerTest : public testing::Test { } template <typename ImmType> - std::string RepeatFRIb(void (Ass::*f)(FPReg, Reg, ImmType), int imm_bits, std::string fmt) { + std::string RepeatFRIb(void (Ass::*f)(FPReg, Reg, ImmType), + int imm_bits, + const std::string& fmt) { return RepeatTemplatedRegistersImmBits<FPReg, Reg, ImmType>(f, imm_bits, GetFPRegisters(), @@ -311,7 +313,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatFF(void (Ass::*f)(FPReg, FPReg), std::string fmt) { + std::string RepeatFF(void (Ass::*f)(FPReg, FPReg), const std::string& fmt) { return RepeatTemplatedRegisters<FPReg, FPReg>(f, GetFPRegisters(), GetFPRegisters(), @@ -320,7 +322,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatFFF(void (Ass::*f)(FPReg, FPReg, FPReg), std::string fmt) { + std::string RepeatFFF(void (Ass::*f)(FPReg, FPReg, FPReg), const std::string& fmt) { return RepeatTemplatedRegisters<FPReg, FPReg, FPReg>(f, GetFPRegisters(), GetFPRegisters(), @@ -331,7 +333,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatFFR(void (Ass::*f)(FPReg, FPReg, Reg), std::string fmt) { + std::string RepeatFFR(void (Ass::*f)(FPReg, FPReg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<FPReg, FPReg, Reg>( f, GetFPRegisters(), @@ -345,7 +347,7 @@ class AssemblerTest : public testing::Test { std::string RepeatFFI(void (Ass::*f)(FPReg, FPReg, const Imm&), size_t imm_bytes, - std::string fmt) { + const std::string& fmt) { return RepeatTemplatedRegistersImm<FPReg, FPReg>(f, GetFPRegisters(), GetFPRegisters(), @@ -356,7 +358,9 @@ class AssemblerTest : public testing::Test { } template <typename ImmType> - std::string RepeatFFIb(void (Ass::*f)(FPReg, FPReg, ImmType), int imm_bits, std::string fmt) { + std::string RepeatFFIb(void (Ass::*f)(FPReg, FPReg, ImmType), + int imm_bits, + const std::string& fmt) { return RepeatTemplatedRegistersImmBits<FPReg, FPReg, ImmType>(f, imm_bits, GetFPRegisters(), @@ -367,7 +371,9 @@ class AssemblerTest : public testing::Test { } template <typename ImmType> - std::string RepeatIbFF(void (Ass::*f)(ImmType, FPReg, FPReg), int imm_bits, std::string fmt) { + std::string RepeatIbFF(void (Ass::*f)(ImmType, FPReg, FPReg), + int imm_bits, + const std::string& fmt) { return RepeatTemplatedImmBitsRegisters<ImmType, FPReg, FPReg>(f, GetFPRegisters(), GetFPRegisters(), @@ -377,7 +383,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatFR(void (Ass::*f)(FPReg, Reg), std::string fmt) { + std::string RepeatFR(void (Ass::*f)(FPReg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<FPReg, Reg>(f, GetFPRegisters(), GetRegisters(), @@ -386,7 +392,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatFr(void (Ass::*f)(FPReg, Reg), std::string fmt) { + std::string RepeatFr(void (Ass::*f)(FPReg, Reg), const std::string& fmt) { return RepeatTemplatedRegisters<FPReg, Reg>(f, GetFPRegisters(), GetRegisters(), @@ -395,7 +401,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatRF(void (Ass::*f)(Reg, FPReg), std::string fmt) { + std::string RepeatRF(void (Ass::*f)(Reg, FPReg), const std::string& fmt) { return RepeatTemplatedRegisters<Reg, FPReg>(f, GetRegisters(), GetFPRegisters(), @@ -404,7 +410,7 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatrF(void (Ass::*f)(Reg, FPReg), std::string fmt) { + std::string RepeatrF(void (Ass::*f)(Reg, FPReg), const std::string& fmt) { return RepeatTemplatedRegisters<Reg, FPReg>(f, GetRegisters(), GetFPRegisters(), @@ -413,7 +419,9 @@ class AssemblerTest : public testing::Test { fmt); } - std::string RepeatI(void (Ass::*f)(const Imm&), size_t imm_bytes, std::string fmt, + std::string RepeatI(void (Ass::*f)(const Imm&), + size_t imm_bytes, + const std::string& fmt, bool as_uint = false) { std::string str; std::vector<int64_t> imms = CreateImmediateValues(imm_bytes, as_uint); @@ -651,7 +659,7 @@ class AssemblerTest : public testing::Test { std::string RepeatTemplatedRegister(void (Ass::*f)(RegType), const std::vector<RegType*> registers, std::string (AssemblerTest::*GetName)(const RegType&), - std::string fmt) { + const std::string& fmt) { std::string str; for (auto reg : registers) { (assembler_.get()->*f)(*reg); @@ -679,7 +687,7 @@ class AssemblerTest : public testing::Test { const std::vector<Reg2*> reg2_registers, std::string (AssemblerTest::*GetName1)(const Reg1&), std::string (AssemblerTest::*GetName2)(const Reg2&), - std::string fmt) { + const std::string& fmt) { WarnOnCombinations(reg1_registers.size() * reg2_registers.size()); std::string str; @@ -717,7 +725,7 @@ class AssemblerTest : public testing::Test { const std::vector<Reg2*> reg2_registers, std::string (AssemblerTest::*GetName1)(const Reg1&), std::string (AssemblerTest::*GetName2)(const Reg2&), - std::string fmt) { + const std::string& fmt) { WarnOnCombinations(reg1_registers.size() * reg2_registers.size()); std::string str; @@ -758,7 +766,7 @@ class AssemblerTest : public testing::Test { std::string (AssemblerTest::*GetName1)(const Reg1&), std::string (AssemblerTest::*GetName2)(const Reg2&), std::string (AssemblerTest::*GetName3)(const Reg3&), - std::string fmt) { + const std::string& fmt) { std::string str; for (auto reg1 : reg1_registers) { for (auto reg2 : reg2_registers) { @@ -803,7 +811,7 @@ class AssemblerTest : public testing::Test { std::string (AssemblerTest::*GetName1)(const Reg1&), std::string (AssemblerTest::*GetName2)(const Reg2&), size_t imm_bytes, - std::string fmt) { + const std::string& fmt) { std::vector<int64_t> imms = CreateImmediateValues(imm_bytes); WarnOnCombinations(reg1_registers.size() * reg2_registers.size() * imms.size()); @@ -895,8 +903,9 @@ class AssemblerTest : public testing::Test { private: template <RegisterView kRegView> - std::string RepeatRegisterImm(void (Ass::*f)(Reg, const Imm&), size_t imm_bytes, - std::string fmt) { + std::string RepeatRegisterImm(void (Ass::*f)(Reg, const Imm&), + size_t imm_bytes, + const std::string& fmt) { const std::vector<Reg*> registers = GetRegisters(); std::string str; std::vector<int64_t> imms = CreateImmediateValues(imm_bytes); @@ -938,7 +947,7 @@ class AssemblerTest : public testing::Test { virtual void Pad(std::vector<uint8_t>& data ATTRIBUTE_UNUSED) { } - void DriverWrapper(std::string assembly_text, std::string test_name) { + void DriverWrapper(const std::string& assembly_text, const std::string& test_name) { assembler_->FinalizeCode(); size_t cs = assembler_->CodeSize(); std::unique_ptr<std::vector<uint8_t>> data(new std::vector<uint8_t>(cs)); diff --git a/compiler/utils/assembler_test_base.h b/compiler/utils/assembler_test_base.h index 8c71292465..ac24ee95eb 100644 --- a/compiler/utils/assembler_test_base.h +++ b/compiler/utils/assembler_test_base.h @@ -106,7 +106,9 @@ class AssemblerTestInfrastructure { // Driver() assembles and compares the results. If the results are not equal and we have a // disassembler, disassemble both and check whether they have the same mnemonics (in which case // we just warn). - void Driver(const std::vector<uint8_t>& data, std::string assembly_text, std::string test_name) { + void Driver(const std::vector<uint8_t>& data, + const std::string& assembly_text, + const std::string& test_name) { EXPECT_NE(assembly_text.length(), 0U) << "Empty assembly"; NativeAssemblerResult res; @@ -229,7 +231,7 @@ class AssemblerTestInfrastructure { bool success = Exec(args, error_msg); if (!success) { LOG(ERROR) << "Assembler command line:"; - for (std::string arg : args) { + for (const std::string& arg : args) { LOG(ERROR) << arg; } } @@ -238,7 +240,7 @@ class AssemblerTestInfrastructure { // Runs objdump -h on the binary file and extracts the first line with .text. // Returns "" on failure. - std::string Objdump(std::string file) { + std::string Objdump(const std::string& file) { bool have_objdump = FileExists(FindTool(objdump_cmd_name_)); EXPECT_TRUE(have_objdump) << "Cannot find objdump: " << GetObjdumpCommand(); if (!have_objdump) { @@ -287,8 +289,9 @@ class AssemblerTestInfrastructure { } // Disassemble both binaries and compare the text. - bool DisassembleBinaries(const std::vector<uint8_t>& data, const std::vector<uint8_t>& as, - std::string test_name) { + bool DisassembleBinaries(const std::vector<uint8_t>& data, + const std::vector<uint8_t>& as, + const std::string& test_name) { std::string disassembler = GetDisassembleCommand(); if (disassembler.length() == 0) { LOG(WARNING) << "No dissassembler command."; @@ -324,7 +327,7 @@ class AssemblerTestInfrastructure { return result; } - bool DisassembleBinary(std::string file, std::string* error_msg) { + bool DisassembleBinary(const std::string& file, std::string* error_msg) { std::vector<std::string> args; // Encaspulate the whole command line in a single string passed to @@ -345,7 +348,7 @@ class AssemblerTestInfrastructure { return Exec(args, error_msg); } - std::string WriteToFile(const std::vector<uint8_t>& buffer, std::string test_name) { + std::string WriteToFile(const std::vector<uint8_t>& buffer, const std::string& test_name) { std::string file_name = GetTmpnam() + std::string("---") + test_name; const char* data = reinterpret_cast<const char*>(buffer.data()); std::ofstream s_out(file_name + ".o"); @@ -354,7 +357,7 @@ class AssemblerTestInfrastructure { return file_name + ".o"; } - bool CompareFiles(std::string f1, std::string f2) { + bool CompareFiles(const std::string& f1, const std::string& f2) { std::ifstream f1_in(f1); std::ifstream f2_in(f2); @@ -369,7 +372,9 @@ class AssemblerTestInfrastructure { } // Compile the given assembly code and extract the binary, if possible. Put result into res. - bool Compile(std::string assembly_code, NativeAssemblerResult* res, std::string test_name) { + bool Compile(const std::string& assembly_code, + NativeAssemblerResult* res, + const std::string& test_name) { res->ok = false; res->code.reset(nullptr); @@ -438,7 +443,7 @@ class AssemblerTestInfrastructure { // Check whether file exists. Is used for commands, so strips off any parameters: anything after // the first space. We skip to the last slash for this, so it should work with directories with // spaces. - static bool FileExists(std::string file) { + static bool FileExists(const std::string& file) { if (file.length() == 0) { return false; } @@ -478,7 +483,7 @@ class AssemblerTestInfrastructure { return getcwd(temp, 1024) ? std::string(temp) + "/" : std::string(""); } - std::string FindTool(std::string tool_name) { + std::string FindTool(const std::string& tool_name) { // Find the current tool. Wild-card pattern is "arch-string*tool-name". std::string gcc_path = GetRootPath() + GetGCCRootPath(); std::vector<std::string> args; @@ -522,7 +527,8 @@ class AssemblerTestInfrastructure { // Helper for below. If name_predicate is empty, search for all files, otherwise use it for the // "-name" option. - static void FindToolDumpPrintout(std::string name_predicate, std::string tmp_file) { + static void FindToolDumpPrintout(const std::string& name_predicate, + const std::string& tmp_file) { std::string gcc_path = GetRootPath() + GetGCCRootPath(); std::vector<std::string> args; args.push_back("find"); @@ -562,7 +568,7 @@ class AssemblerTestInfrastructure { } // For debug purposes. - void FindToolDump(std::string tool_name) { + void FindToolDump(const std::string& tool_name) { // Check with the tool name. FindToolDumpPrintout(architecture_string_ + "*" + tool_name, GetTmpnam()); FindToolDumpPrintout("", GetTmpnam()); diff --git a/compiler/utils/jni_macro_assembler_test.h b/compiler/utils/jni_macro_assembler_test.h index 829f34b4b7..293f4cde9c 100644 --- a/compiler/utils/jni_macro_assembler_test.h +++ b/compiler/utils/jni_macro_assembler_test.h @@ -39,12 +39,12 @@ class JNIMacroAssemblerTest : public testing::Test { typedef std::string (*TestFn)(JNIMacroAssemblerTest* assembler_test, Ass* assembler); - void DriverFn(TestFn f, std::string test_name) { + void DriverFn(TestFn f, const std::string& test_name) { DriverWrapper(f(this, assembler_.get()), test_name); } // This driver assumes the assembler has already been called. - void DriverStr(std::string assembly_string, std::string test_name) { + void DriverStr(const std::string& assembly_string, const std::string& test_name) { DriverWrapper(assembly_string, test_name); } @@ -128,7 +128,7 @@ class JNIMacroAssemblerTest : public testing::Test { virtual void Pad(std::vector<uint8_t>& data ATTRIBUTE_UNUSED) { } - void DriverWrapper(std::string assembly_text, std::string test_name) { + void DriverWrapper(const std::string& assembly_text, const std::string& test_name) { assembler_->FinalizeCode(); size_t cs = assembler_->CodeSize(); std::unique_ptr<std::vector<uint8_t>> data(new std::vector<uint8_t>(cs)); diff --git a/compiler/utils/managed_register.h b/compiler/utils/managed_register.h index 46adb3f2d2..184cdf5050 100644 --- a/compiler/utils/managed_register.h +++ b/compiler/utils/managed_register.h @@ -17,8 +17,11 @@ #ifndef ART_COMPILER_UTILS_MANAGED_REGISTER_H_ #define ART_COMPILER_UTILS_MANAGED_REGISTER_H_ +#include <type_traits> #include <vector> +#include "base/value_object.h" + namespace art { namespace arm { @@ -42,17 +45,14 @@ namespace x86_64 { class X86_64ManagedRegister; } -class ManagedRegister { +class ManagedRegister : public ValueObject { public: // ManagedRegister is a value class. There exists no method to change the // internal state. We therefore allow a copy constructor and an // assignment-operator. - constexpr ManagedRegister(const ManagedRegister& other) : id_(other.id_) { } + constexpr ManagedRegister(const ManagedRegister& other) = default; - ManagedRegister& operator=(const ManagedRegister& other) { - id_ = other.id_; - return *this; - } + ManagedRegister& operator=(const ManagedRegister& other) = default; constexpr arm::ArmManagedRegister AsArm() const; constexpr arm64::Arm64ManagedRegister AsArm64() const; @@ -85,6 +85,9 @@ class ManagedRegister { int id_; }; +static_assert(std::is_trivially_copyable<ManagedRegister>::value, + "ManagedRegister should be trivially copyable"); + class ManagedRegisterSpill : public ManagedRegister { public: // ManagedRegisterSpill contains information about data type size and location in caller frame @@ -115,18 +118,18 @@ class ManagedRegisterEntrySpills : public std::vector<ManagedRegisterSpill> { public: // The ManagedRegister does not have information about size and offset. // In this case it's size and offset determined by BuildFrame (assembler) - void push_back(ManagedRegister __x) { - ManagedRegisterSpill spill(__x); + void push_back(ManagedRegister x) { + ManagedRegisterSpill spill(x); std::vector<ManagedRegisterSpill>::push_back(spill); } - void push_back(ManagedRegister __x, int32_t __size) { - ManagedRegisterSpill spill(__x, __size); + void push_back(ManagedRegister x, int32_t size) { + ManagedRegisterSpill spill(x, size); std::vector<ManagedRegisterSpill>::push_back(spill); } - void push_back(ManagedRegisterSpill __x) { - std::vector<ManagedRegisterSpill>::push_back(__x); + void push_back(ManagedRegisterSpill x) { + std::vector<ManagedRegisterSpill>::push_back(x); } private: }; diff --git a/compiler/utils/mips/assembler_mips32r6_test.cc b/compiler/utils/mips/assembler_mips32r6_test.cc index 3ef2f9440c..a52f519439 100644 --- a/compiler/utils/mips/assembler_mips32r6_test.cc +++ b/compiler/utils/mips/assembler_mips32r6_test.cc @@ -219,7 +219,7 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, void BranchCondTwoRegsHelper(void (mips::MipsAssembler::*f)(mips::Register, mips::Register, mips::MipsLabel*), - std::string instr_name) { + const std::string& instr_name) { mips::MipsLabel label; (Base::GetAssembler()->*f)(mips::A0, mips::A1, &label); constexpr size_t kAdduCount1 = 63; diff --git a/compiler/utils/mips/assembler_mips_test.cc b/compiler/utils/mips/assembler_mips_test.cc index 75149cf242..c24e1b16fb 100644 --- a/compiler/utils/mips/assembler_mips_test.cc +++ b/compiler/utils/mips/assembler_mips_test.cc @@ -188,7 +188,7 @@ class AssemblerMIPSTest : public AssemblerTest<mips::MipsAssembler, void BranchCondOneRegHelper(void (mips::MipsAssembler::*f)(mips::Register, mips::MipsLabel*), - std::string instr_name) { + const std::string& instr_name) { mips::MipsLabel label; (Base::GetAssembler()->*f)(mips::A0, &label); constexpr size_t kAdduCount1 = 63; @@ -217,7 +217,7 @@ class AssemblerMIPSTest : public AssemblerTest<mips::MipsAssembler, void BranchCondTwoRegsHelper(void (mips::MipsAssembler::*f)(mips::Register, mips::Register, mips::MipsLabel*), - std::string instr_name) { + const std::string& instr_name) { mips::MipsLabel label; (Base::GetAssembler()->*f)(mips::A0, mips::A1, &label); constexpr size_t kAdduCount1 = 63; diff --git a/compiler/utils/mips64/assembler_mips64_test.cc b/compiler/utils/mips64/assembler_mips64_test.cc index 1fdef96fe4..ba8f25ea77 100644 --- a/compiler/utils/mips64/assembler_mips64_test.cc +++ b/compiler/utils/mips64/assembler_mips64_test.cc @@ -212,7 +212,7 @@ class AssemblerMIPS64Test : public AssemblerTest<mips64::Mips64Assembler, void BranchCondOneRegHelper(void (mips64::Mips64Assembler::*f)(mips64::GpuRegister, mips64::Mips64Label*), - std::string instr_name) { + const std::string& instr_name) { mips64::Mips64Label label; (Base::GetAssembler()->*f)(mips64::A0, &label); constexpr size_t kAdduCount1 = 63; @@ -241,7 +241,7 @@ class AssemblerMIPS64Test : public AssemblerTest<mips64::Mips64Assembler, void BranchCondTwoRegsHelper(void (mips64::Mips64Assembler::*f)(mips64::GpuRegister, mips64::GpuRegister, mips64::Mips64Label*), - std::string instr_name) { + const std::string& instr_name) { mips64::Mips64Label label; (Base::GetAssembler()->*f)(mips64::A0, mips64::A1, &label); constexpr size_t kAdduCount1 = 63; diff --git a/compiler/verifier_deps_test.cc b/compiler/verifier_deps_test.cc index 8d2a0e7860..6b690aab40 100644 --- a/compiler/verifier_deps_test.cc +++ b/compiler/verifier_deps_test.cc @@ -72,6 +72,22 @@ class VerifierDepsTest : public CommonCompilerTest { return klass; } + void SetupCompilerDriver() { + compiler_options_->boot_image_ = false; + compiler_driver_->InitializeThreadPools(); + } + + void VerifyWithCompilerDriver(verifier::VerifierDeps* deps) { + TimingLogger timings("Verify", false, false); + // The compiler driver handles the verifier deps in the callbacks, so + // remove what this class did for unit testing. + verifier_deps_.reset(nullptr); + callbacks_->SetVerifierDeps(nullptr); + compiler_driver_->Verify(class_loader_, dex_files_, deps, &timings); + // The compiler driver may have updated the VerifierDeps in the callback object. + verifier_deps_.reset(callbacks_->GetVerifierDeps()); + } + void SetVerifierDeps(const std::vector<const DexFile*>& dex_files) { verifier_deps_.reset(new verifier::VerifierDeps(dex_files)); VerifierDepsCompilerCallbacks* callbacks = @@ -156,15 +172,12 @@ class VerifierDepsTest : public CommonCompilerTest { } void VerifyDexFile(const char* multidex = nullptr) { - std::string error_msg; { ScopedObjectAccess soa(Thread::Current()); LoadDexFile(&soa, "VerifierDeps", multidex); } - TimingLogger timings("Verify", false, false); - compiler_options_->boot_image_ = false; - compiler_driver_->InitializeThreadPools(); - compiler_driver_->Verify(class_loader_, dex_files_, &timings); + SetupCompilerDriver(); + VerifyWithCompilerDriver(/* verifier_deps */ nullptr); } bool TestAssignabilityRecording(const std::string& dst, @@ -185,6 +198,33 @@ class VerifierDepsTest : public CommonCompilerTest { return true; } + // Check that the status of classes in `class_loader_` match the + // expected status in `deps`. + void VerifyClassStatus(const verifier::VerifierDeps& deps) { + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<2> hs(soa.Self()); + Handle<mirror::ClassLoader> class_loader_handle( + hs.NewHandle(soa.Decode<mirror::ClassLoader>(class_loader_))); + MutableHandle<mirror::Class> cls(hs.NewHandle<mirror::Class>(nullptr)); + for (const DexFile* dex_file : dex_files_) { + const std::vector<uint16_t>& unverified_classes = deps.GetUnverifiedClasses(*dex_file); + std::set<uint16_t> set(unverified_classes.begin(), unverified_classes.end()); + for (uint32_t i = 0; i < dex_file->NumClassDefs(); ++i) { + const DexFile::ClassDef& class_def = dex_file->GetClassDef(i); + const char* descriptor = dex_file->GetClassDescriptor(class_def); + cls.Assign(class_linker_->FindClass(soa.Self(), descriptor, class_loader_handle)); + if (cls.Get() == nullptr) { + CHECK(soa.Self()->IsExceptionPending()); + soa.Self()->ClearException(); + } else if (set.find(class_def.class_idx_) == set.end()) { + ASSERT_EQ(cls->GetStatus(), mirror::Class::kStatusVerified); + } else { + ASSERT_LT(cls->GetStatus(), mirror::Class::kStatusVerified); + } + } + } + } + bool HasUnverifiedClass(const std::string& cls) { const DexFile::TypeId* type_id = primary_dex_file_->FindTypeId(cls.c_str()); DCHECK(type_id != nullptr); @@ -1160,7 +1200,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { MutableHandle<mirror::ClassLoader> new_class_loader(hs.NewHandle<mirror::ClassLoader>(nullptr)); { new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_TRUE(verifier_deps_->Verify(new_class_loader, soa.Self())); + ASSERT_TRUE(verifier_deps_->ValidateDependencies(new_class_loader, soa.Self())); } std::vector<uint8_t> buffer; @@ -1170,7 +1210,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { { VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_TRUE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_TRUE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } // Fiddle with the dependencies to make sure we catch any change and fail to verify. @@ -1181,7 +1221,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); deps->assignable_types_.insert(*deps->unassignable_types_.begin()); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1190,7 +1230,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); deps->unassignable_types_.insert(*deps->assignable_types_.begin()); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } // Mess up with classes. @@ -1208,7 +1248,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1225,7 +1265,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1242,7 +1282,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } // Mess up with fields. @@ -1261,7 +1301,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1279,7 +1319,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1297,7 +1337,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1316,7 +1356,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } // Mess up with methods. @@ -1338,7 +1378,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1357,7 +1397,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1376,7 +1416,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1396,7 +1436,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1415,7 +1455,7 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); } { @@ -1434,7 +1474,56 @@ TEST_F(VerifierDepsTest, VerifyDeps) { } ASSERT_TRUE(found); new_class_loader.Assign(soa.Decode<mirror::ClassLoader>(LoadDex("VerifierDeps"))); - ASSERT_FALSE(decoded_deps.Verify(new_class_loader, soa.Self())); + ASSERT_FALSE(decoded_deps.ValidateDependencies(new_class_loader, soa.Self())); + } + } +} + +TEST_F(VerifierDepsTest, CompilerDriver) { + SetupCompilerDriver(); + + // Test both multi-dex and single-dex configuration. + for (const char* multi : { "MultiDex", static_cast<const char*>(nullptr) }) { + // Test that the compiler driver behaves as expected when the dependencies + // verify and when they don't verify. + for (bool verify_failure : { false, true }) { + { + ScopedObjectAccess soa(Thread::Current()); + LoadDexFile(&soa, "VerifierDeps", multi); + } + VerifyWithCompilerDriver(/* verifier_deps */ nullptr); + + std::vector<uint8_t> buffer; + verifier_deps_->Encode(dex_files_, &buffer); + + { + ScopedObjectAccess soa(Thread::Current()); + LoadDexFile(&soa, "VerifierDeps", multi); + } + verifier::VerifierDeps decoded_deps(dex_files_, ArrayRef<const uint8_t>(buffer)); + if (verify_failure) { + // Just taint the decoded VerifierDeps with one invalid entry. + VerifierDeps::DexFileDeps* deps = decoded_deps.GetDexFileDeps(*primary_dex_file_); + bool found = false; + for (const auto& entry : deps->classes_) { + if (entry.IsResolved()) { + deps->classes_.insert(VerifierDeps::ClassResolution( + entry.GetDexTypeIndex(), VerifierDeps::kUnresolvedMarker)); + found = true; + break; + } + } + ASSERT_TRUE(found); + } + VerifyWithCompilerDriver(&decoded_deps); + + if (verify_failure) { + ASSERT_FALSE(verifier_deps_ == nullptr); + ASSERT_FALSE(verifier_deps_->Equals(decoded_deps)); + } else { + ASSERT_TRUE(verifier_deps_ == nullptr); + VerifyClassStatus(decoded_deps); + } } } } diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index 28d628920c..1180bdeabb 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -1497,12 +1497,6 @@ class Dex2Oat FINAL { dex_files_ = MakeNonOwningPointerVector(opened_dex_files_); - if (!IsBootImage()) { - // Collect verification dependencies when compiling an app. - verifier_deps_.reset(new verifier::VerifierDeps(dex_files_)); - callbacks_->SetVerifierDeps(verifier_deps_.get()); - } - // We had to postpone the swap decision till now, as this is the point when we actually // know about the dex files we're going to use. @@ -1660,7 +1654,7 @@ class Dex2Oat FINAL { swap_fd_, profile_compilation_info_.get())); driver_->SetDexFilesForOatFile(dex_files_); - driver_->CompileAll(class_loader_, dex_files_, timings_); + driver_->CompileAll(class_loader_, dex_files_, /* verifier_deps */ nullptr, timings_); } // Notes on the interleaving of creating the images and oat files to @@ -1785,13 +1779,13 @@ class Dex2Oat FINAL { { TimingLogger::ScopedTiming t2("dex2oat Write VDEX", timings_); DCHECK(IsBootImage() || oat_files_.size() == 1u); - DCHECK_EQ(IsBootImage(), verifier_deps_ == nullptr); + verifier::VerifierDeps* verifier_deps = callbacks_->GetVerifierDeps(); for (size_t i = 0, size = oat_files_.size(); i != size; ++i) { File* vdex_file = vdex_files_[i].get(); std::unique_ptr<BufferedOutputStream> vdex_out( MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(vdex_file))); - if (!oat_writers_[i]->WriteVerifierDeps(vdex_out.get(), verifier_deps_.get())) { + if (!oat_writers_[i]->WriteVerifierDeps(vdex_out.get(), verifier_deps)) { LOG(ERROR) << "Failed to write verifier dependencies into VDEX " << vdex_file->GetPath(); return false; } @@ -2645,9 +2639,6 @@ class Dex2Oat FINAL { std::vector<std::vector<const DexFile*>> dex_files_per_oat_file_; std::unordered_map<const DexFile*, size_t> dex_file_oat_index_map_; - // Collector of verifier dependencies. - std::unique_ptr<verifier::VerifierDeps> verifier_deps_; - // Backing storage. std::vector<std::string> char_backing_storage_; diff --git a/imgdiag/imgdiag.cc b/imgdiag/imgdiag.cc index f197fc110f..d1d127d980 100644 --- a/imgdiag/imgdiag.cc +++ b/imgdiag/imgdiag.cc @@ -516,8 +516,8 @@ class ImgDiagDumper { // Sanity check that we are reading a real object CHECK(obj->GetClass() != nullptr) << "Image object at address " << obj << " has null class"; - if (kUseBakerOrBrooksReadBarrier) { - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); } // Iterate every page this object belongs to diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc index 986f265b3f..7ea5beab37 100644 --- a/patchoat/patchoat.cc +++ b/patchoat/patchoat.cc @@ -747,13 +747,8 @@ void PatchOat::PatchVisitor::operator() (ObjPtr<mirror::Class> cls ATTRIBUTE_UNU void PatchOat::VisitObject(mirror::Object* object) { mirror::Object* copy = RelocatedCopyOf(object); CHECK(copy != nullptr); - if (kUseBakerOrBrooksReadBarrier) { - object->AssertReadBarrierPointer(); - if (kUseBrooksReadBarrier) { - mirror::Object* moved_to = RelocatedAddressOfPointer(object); - copy->SetReadBarrierPointer(moved_to); - DCHECK_EQ(copy->GetReadBarrierPointer(), moved_to); - } + if (kUseBakerReadBarrier) { + object->AssertReadBarrierState(); } PatchOat::PatchVisitor visitor(this, copy); object->VisitReferences<kVerifyNone>(visitor, visitor); diff --git a/runtime/Android.bp b/runtime/Android.bp index 9157907cee..c6f479ff40 100644 --- a/runtime/Android.bp +++ b/runtime/Android.bp @@ -120,6 +120,7 @@ cc_defaults { "linear_alloc.cc", "mem_map.cc", "memory_region.cc", + "method_handles.cc", "mirror/array.cc", "mirror/class.cc", "mirror/class_ext.cc", @@ -405,7 +406,8 @@ cc_defaults { gensrcs { name: "art_operator_srcs", - cmd: "art/tools/generate-operator-out.py art/runtime $in > $out", + cmd: "$(location generate-operator-out.py) art/runtime $(in) > $(out)", + tool_files: ["generate-operator-out.py"], srcs: [ "arch/instruction_set.h", "base/allocator.h", diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index bf70c554b1..0135260de8 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -1999,11 +1999,17 @@ ENTRY \name // Check lock word for mark bit, if marked return. Use IP for scratch since it is blocked. ldr ip, [\reg, MIRROR_OBJECT_LOCK_WORD_OFFSET] tst ip, #LOCK_WORD_MARK_BIT_MASK_SHIFTED - beq .Lslow_rb_\name + beq .Lnot_marked_rb_\name // Already marked, return right away. .Lret_rb_\name: bx lr +.Lnot_marked_rb_\name: + // Test that both the forwarding state bits are 1. + mvn ip, ip + tst ip, #(LOCK_WORD_STATE_FORWARDING_ADDRESS << LOCK_WORD_STATE_SHIFT) + beq .Lret_forwarding_address\name + .Lslow_rb_\name: // Save IP: the kSaveEverything entrypoint art_quick_resolve_string makes a tail call here. push {r0-r4, r9, ip, lr} @ save return address, core caller-save registers and ip @@ -2064,6 +2070,12 @@ ENTRY \name .cfi_restore ip .cfi_restore lr bx lr +.Lret_forwarding_address\name: + // Shift left by the forwarding address shift. This clears out the state bits since they are + // in the top 2 bits of the lock word. + mvn ip, ip + lsl \reg, ip, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT + bx lr END \name .endm diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index 483cee3100..d806715ec9 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -2539,10 +2539,17 @@ ENTRY \name */ // Use wIP0 as temp and check the mark bit of the reference. wIP0 is not used by the compiler. ldr wIP0, [\xreg, #MIRROR_OBJECT_LOCK_WORD_OFFSET] - tbz wIP0, #LOCK_WORD_MARK_BIT_SHIFT, .Lslow_rb_\name + tbz wIP0, #LOCK_WORD_MARK_BIT_SHIFT, .Lnot_marked_rb_\name .Lret_rb_\name: ret +.Lnot_marked_rb_\name: + // Check if the top two bits are one, if this is the case it is a forwarding address. + mvn wIP0, wIP0 + cmp wzr, wIP0, lsr #30 + beq .Lret_forwarding_address\name .Lslow_rb_\name: + // We must not clobber IP0 since art_quick_resolve_string makes a tail call here and relies on + // IP0 being restored. // Save all potentially live caller-save core registers. SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 368 SAVE_TWO_REGS x2, x3, 16 @@ -2608,6 +2615,12 @@ ENTRY \name RESTORE_REG xLR, 360 DECREASE_FRAME 368 ret +.Lret_forwarding_address\name: + mvn wIP0, wIP0 + // Shift left by the forwarding address shift. This clears out the state bits since they are + // in the top 2 bits of the lock word. + lsl \wreg, wIP0, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT + ret END \name .endm diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S index f4f9a68e30..98739d3cc8 100644 --- a/runtime/arch/x86/quick_entrypoints_x86.S +++ b/runtime/arch/x86/quick_entrypoints_x86.S @@ -2155,8 +2155,15 @@ MACRO2(READ_BARRIER_MARK_REG, name, reg) jz .Lslow_rb_\name ret .Lslow_rb_\name: - // Save all potentially live caller-save core registers. PUSH eax + mov MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg)), %eax + add LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW), %eax + // Jump if overflow, the only case where it overflows should be the forwarding address one. + // Taken ~25% of the time. + jnae .Lret_forwarding_address\name + + // Save all potentially live caller-save core registers. + mov 0(%esp), %eax PUSH ecx PUSH edx PUSH ebx @@ -2204,6 +2211,12 @@ MACRO2(READ_BARRIER_MARK_REG, name, reg) POP_REG_NE eax, RAW_VAR(reg) .Lret_rb_\name: ret +.Lret_forwarding_address\name: + // The overflow cleared the top bits. + sall LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT), %eax + mov %eax, REG_VAR(reg) + POP_REG_NE eax, RAW_VAR(reg) + ret END_FUNCTION VAR(name) END_MACRO diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S index 2856766fec..185e55e114 100644 --- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S +++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S @@ -1481,31 +1481,6 @@ DEFINE_FUNCTION art_quick_unlock_object_no_inline END_FUNCTION art_quick_unlock_object_no_inline DEFINE_FUNCTION art_quick_check_cast - testl LITERAL(ACCESS_FLAGS_CLASS_IS_INTERFACE), MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%rdi) - jz .Lnot_interface - - // There are no read barriers since the iftable is immutable. There can be false negatives for - // the read barrier case if classes in the IfTable are in the from-space. In the case where - // we do not find a matching interface we call into artIsAssignableFromCode which will have - // read barriers. - movl MIRROR_CLASS_IF_TABLE_OFFSET(%rsi), %ecx - UNPOISON_HEAP_REF ecx - testl %ecx, %ecx - jz .Lnot_interface - movl MIRROR_ARRAY_LENGTH_OFFSET(%rcx), %r8d -.Lstart_loop: - // Re-poison before comparing to prevent rare possible false positives. This is done inside - // the loop since heap poisoning is only for testing builds. - POISON_HEAP_REF edi - cmpl MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rcx), %edi - je .Lreturn // Return if same class. - UNPOISON_HEAP_REF edi - // Go to next interface. - add LITERAL(COMPRESSED_REFERENCE_SIZE * 2), %rcx - sub LITERAL(2), %r8 - jnz .Lstart_loop - -.Lnot_interface: // We could check the super classes here but that is usually already checked in the caller. PUSH rdi // Save args for exc PUSH rsi @@ -2301,8 +2276,16 @@ MACRO2(READ_BARRIER_MARK_REG, name, reg) jz .Lslow_rb_\name ret .Lslow_rb_\name: - // Save all potentially live caller-save core registers. PUSH rax + movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg)), %eax + addl LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW), %eax + // Jump if the addl caused eax to unsigned overflow. The only case where it overflows is the + // forwarding address one. + // Taken ~25% of the time. + jnae .Lret_forwarding_address\name + + // Save all potentially live caller-save core registers. + movq 0(%rsp), %rax PUSH rcx PUSH rdx PUSH rsi @@ -2367,6 +2350,12 @@ MACRO2(READ_BARRIER_MARK_REG, name, reg) POP_REG_NE rax, RAW_VAR(reg) .Lret_rb_\name: ret +.Lret_forwarding_address\name: + // The overflow cleared the top bits. + sall LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT), %eax + movq %rax, REG_VAR(reg) + POP_REG_NE rax, RAW_VAR(reg) + ret END_FUNCTION VAR(name) END_MACRO diff --git a/runtime/base/mutex.cc b/runtime/base/mutex.cc index bde03277b7..5d922989d9 100644 --- a/runtime/base/mutex.cc +++ b/runtime/base/mutex.cc @@ -42,7 +42,6 @@ Mutex* Locks::deoptimization_lock_ = nullptr; ReaderWriterMutex* Locks::heap_bitmap_lock_ = nullptr; Mutex* Locks::instrument_entrypoints_lock_ = nullptr; Mutex* Locks::intern_table_lock_ = nullptr; -Mutex* Locks::interpreter_string_init_map_lock_ = nullptr; Mutex* Locks::jni_libraries_lock_ = nullptr; Mutex* Locks::logging_lock_ = nullptr; Mutex* Locks::mem_maps_lock_ = nullptr; diff --git a/runtime/base/mutex.h b/runtime/base/mutex.h index 3f2c5a9e74..74b786c38e 100644 --- a/runtime/base/mutex.h +++ b/runtime/base/mutex.h @@ -79,7 +79,6 @@ enum LockLevel { kAllocSpaceLock, kBumpPointerSpaceBlockLock, kArenaPoolLock, - kDexFileToMethodInlinerMapLock, kInternTableLock, kOatFileSecondaryLookupLock, kHostDlOpenHandlesLock, @@ -92,12 +91,10 @@ enum LockLevel { kDefaultMutexLevel, kDexLock, kMarkSweepLargeObjectLock, - kPinTableLock, kJdwpObjectRegistryLock, kModifyLdtLock, kAllocatedThreadIdsLock, kMonitorPoolLock, - kMethodVerifiersLock, kClassLinkerClassesLock, // TODO rename. kJitCodeCacheLock, kBreakpointLock, @@ -630,12 +627,9 @@ class Locks { // TODO: improve name, perhaps instrumentation_update_lock_. static Mutex* deoptimization_lock_ ACQUIRED_AFTER(alloc_tracker_lock_); - // Guards String initializer register map in interpreter. - static Mutex* interpreter_string_init_map_lock_ ACQUIRED_AFTER(deoptimization_lock_); - // The thread_list_lock_ guards ThreadList::list_. It is also commonly held to stop threads // attaching and detaching. - static Mutex* thread_list_lock_ ACQUIRED_AFTER(interpreter_string_init_map_lock_); + static Mutex* thread_list_lock_ ACQUIRED_AFTER(deoptimization_lock_); // Signaled when threads terminate. Used to determine when all non-daemons have terminated. static ConditionVariable* thread_exit_cond_ GUARDED_BY(Locks::thread_list_lock_); diff --git a/runtime/base/unix_file/fd_file.cc b/runtime/base/unix_file/fd_file.cc index 4498198b34..ff2dd1b399 100644 --- a/runtime/base/unix_file/fd_file.cc +++ b/runtime/base/unix_file/fd_file.cc @@ -339,22 +339,59 @@ bool FdFile::Copy(FdFile* input_file, int64_t offset, int64_t size) { return true; } -void FdFile::Erase() { +bool FdFile::Unlink() { + if (file_path_.empty()) { + return false; + } + + // Try to figure out whether this file is still referring to the one on disk. + bool is_current = false; + { + struct stat this_stat, current_stat; + int cur_fd = TEMP_FAILURE_RETRY(open(file_path_.c_str(), O_RDONLY)); + if (cur_fd > 0) { + // File still exists. + if (fstat(fd_, &this_stat) == 0 && fstat(cur_fd, ¤t_stat) == 0) { + is_current = (this_stat.st_dev == current_stat.st_dev) && + (this_stat.st_ino == current_stat.st_ino); + } + close(cur_fd); + } + } + + if (is_current) { + unlink(file_path_.c_str()); + } + + return is_current; +} + +bool FdFile::Erase(bool unlink) { DCHECK(!read_only_mode_); - TEMP_FAILURE_RETRY(SetLength(0)); - TEMP_FAILURE_RETRY(Flush()); - TEMP_FAILURE_RETRY(Close()); + + bool ret_result = true; + if (unlink) { + ret_result = Unlink(); + } + + int result; + result = SetLength(0); + result = Flush(); + result = Close(); + // Ignore the errors. + + return ret_result; } int FdFile::FlushCloseOrErase() { DCHECK(!read_only_mode_); - int flush_result = TEMP_FAILURE_RETRY(Flush()); + int flush_result = Flush(); if (flush_result != 0) { LOG(ERROR) << "CloseOrErase failed while flushing a file."; Erase(); return flush_result; } - int close_result = TEMP_FAILURE_RETRY(Close()); + int close_result = Close(); if (close_result != 0) { LOG(ERROR) << "CloseOrErase failed while closing a file."; Erase(); @@ -365,11 +402,11 @@ int FdFile::FlushCloseOrErase() { int FdFile::FlushClose() { DCHECK(!read_only_mode_); - int flush_result = TEMP_FAILURE_RETRY(Flush()); + int flush_result = Flush(); if (flush_result != 0) { LOG(ERROR) << "FlushClose failed while flushing a file."; } - int close_result = TEMP_FAILURE_RETRY(Close()); + int close_result = Close(); if (close_result != 0) { LOG(ERROR) << "FlushClose failed while closing a file."; } diff --git a/runtime/base/unix_file/fd_file.h b/runtime/base/unix_file/fd_file.h index d896ee9ecb..eb85c4f097 100644 --- a/runtime/base/unix_file/fd_file.h +++ b/runtime/base/unix_file/fd_file.h @@ -97,7 +97,14 @@ class FdFile : public RandomAccessFile { int Flush() OVERRIDE WARN_UNUSED; // Short for SetLength(0); Flush(); Close(); - void Erase(); + // If the file was opened with a path name and unlink = true, also calls Unlink() on the path. + // Note that it is the the caller's responsibility to avoid races. + bool Erase(bool unlink = false); + + // Call unlink() if the file was opened with a path, and if open() with the name shows that + // the file descriptor of this file is still up-to-date. This is still racy, though, and it + // is up to the caller to ensure correctness in a multi-process setup. + bool Unlink(); // Try to Flush(), then try to Close(); If either fails, call Erase(). int FlushCloseOrErase() WARN_UNUSED; diff --git a/runtime/base/unix_file/fd_file_test.cc b/runtime/base/unix_file/fd_file_test.cc index 99ef6f73ba..7657a38cec 100644 --- a/runtime/base/unix_file/fd_file_test.cc +++ b/runtime/base/unix_file/fd_file_test.cc @@ -186,4 +186,24 @@ TEST_F(FdFileTest, MoveConstructor) { ASSERT_EQ(file2.Close(), 0); } +TEST_F(FdFileTest, EraseWithPathUnlinks) { + // New scratch file, zero-length. + art::ScratchFile tmp; + std::string filename = tmp.GetFilename(); + tmp.Close(); // This is required because of the unlink race between the scratch file and the + // FdFile, which leads to close-guard breakage. + FdFile file(filename, O_RDWR, false); + ASSERT_TRUE(file.IsOpened()); + EXPECT_GE(file.Fd(), 0); + uint8_t buffer[16] = { 0 }; + EXPECT_TRUE(file.WriteFully(&buffer, sizeof(buffer))); + EXPECT_EQ(file.Flush(), 0); + + EXPECT_TRUE(file.Erase(true)); + + EXPECT_FALSE(file.IsOpened()); + + EXPECT_FALSE(art::OS::FileExists(filename.c_str())) << filename; +} + } // namespace unix_file diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index cab9d3036f..e7e5be7b75 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -394,8 +394,8 @@ bool ClassLinker::InitWithoutImage(std::vector<std::unique_ptr<const DexFile>> b CHECK(java_lang_Class.Get() != nullptr); mirror::Class::SetClassClass(java_lang_Class.Get()); java_lang_Class->SetClass(java_lang_Class.Get()); - if (kUseBakerOrBrooksReadBarrier) { - java_lang_Class->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + java_lang_Class->AssertReadBarrierState(); } java_lang_Class->SetClassSize(class_class_size); java_lang_Class->SetPrimitiveType(Primitive::kPrimNot); diff --git a/runtime/class_linker.h b/runtime/class_linker.h index 669249fb3c..1d29e319d5 100644 --- a/runtime/class_linker.h +++ b/runtime/class_linker.h @@ -418,10 +418,10 @@ class ClassLinker { REQUIRES(!dex_lock_); void VisitClassRoots(RootVisitor* visitor, VisitRootFlags flags) - REQUIRES(!Locks::classlinker_classes_lock_) + REQUIRES(!Locks::classlinker_classes_lock_, !Locks::trace_lock_) REQUIRES_SHARED(Locks::mutator_lock_); void VisitRoots(RootVisitor* visitor, VisitRootFlags flags) - REQUIRES(!dex_lock_) + REQUIRES(!dex_lock_, !Locks::classlinker_classes_lock_, !Locks::trace_lock_) REQUIRES_SHARED(Locks::mutator_lock_); mirror::DexCache* FindDexCache(Thread* self, diff --git a/runtime/class_linker_test.cc b/runtime/class_linker_test.cc index 7a3ebad082..ab2d9d0a95 100644 --- a/runtime/class_linker_test.cc +++ b/runtime/class_linker_test.cc @@ -742,8 +742,8 @@ struct MethodHandleImplOffsets : public CheckOffsets<mirror::MethodHandleImpl> { MethodHandleImplOffsets() : CheckOffsets<mirror::MethodHandleImpl>( false, "Ljava/lang/invoke/MethodHandle;") { addOffset(OFFSETOF_MEMBER(mirror::MethodHandleImpl, art_field_or_method_), "artFieldOrMethod"); - addOffset(OFFSETOF_MEMBER(mirror::MethodHandleImpl, as_type_cache_), "asTypeCache"); addOffset(OFFSETOF_MEMBER(mirror::MethodHandleImpl, handle_kind_), "handleKind"); + addOffset(OFFSETOF_MEMBER(mirror::MethodHandleImpl, nominal_type_), "nominalType"); addOffset(OFFSETOF_MEMBER(mirror::MethodHandleImpl, method_type_), "type"); } }; diff --git a/runtime/common_runtime_test.h b/runtime/common_runtime_test.h index 827e85e9b6..17e3729a20 100644 --- a/runtime/common_runtime_test.h +++ b/runtime/common_runtime_test.h @@ -52,7 +52,7 @@ class ScratchFile { ScratchFile(const ScratchFile& other, const char* suffix); - explicit ScratchFile(ScratchFile&& other); + ScratchFile(ScratchFile&& other); ScratchFile& operator=(ScratchFile&& other); diff --git a/runtime/compiler_callbacks.h b/runtime/compiler_callbacks.h index 00dedef1e7..806653a265 100644 --- a/runtime/compiler_callbacks.h +++ b/runtime/compiler_callbacks.h @@ -47,6 +47,7 @@ class CompilerCallbacks { virtual bool IsRelocationPossible() = 0; virtual verifier::VerifierDeps* GetVerifierDeps() const = 0; + virtual void SetVerifierDeps(verifier::VerifierDeps* deps ATTRIBUTE_UNUSED) {} bool IsBootImage() { return mode_ == CallbackMode::kCompileBootImage; diff --git a/runtime/dex_file_verifier.cc b/runtime/dex_file_verifier.cc index 7b1fb95d35..be25803f59 100644 --- a/runtime/dex_file_verifier.cc +++ b/runtime/dex_file_verifier.cc @@ -456,22 +456,22 @@ uint32_t DexFileVerifier::ReadUnsignedLittleEndian(uint32_t size) { #define DECODE_UNSIGNED_CHECKED_FROM_WITH_ERROR_VALUE(ptr, var, error_value) \ uint32_t var; \ - if (!DecodeUnsignedLeb128Checked(&ptr, begin_ + size_, &var)) { \ + if (!DecodeUnsignedLeb128Checked(&(ptr), begin_ + size_, &(var))) { \ return error_value; \ } -#define DECODE_UNSIGNED_CHECKED_FROM(ptr, var) \ - uint32_t var; \ - if (!DecodeUnsignedLeb128Checked(&ptr, begin_ + size_, &var)) { \ - ErrorStringPrintf("Read out of bounds"); \ - return false; \ +#define DECODE_UNSIGNED_CHECKED_FROM(ptr, var) \ + uint32_t var; \ + if (!DecodeUnsignedLeb128Checked(&(ptr), begin_ + size_, &(var))) { \ + ErrorStringPrintf("Read out of bounds"); \ + return false; \ } -#define DECODE_SIGNED_CHECKED_FROM(ptr, var) \ - int32_t var; \ - if (!DecodeSignedLeb128Checked(&ptr, begin_ + size_, &var)) { \ - ErrorStringPrintf("Read out of bounds"); \ - return false; \ +#define DECODE_SIGNED_CHECKED_FROM(ptr, var) \ + int32_t var; \ + if (!DecodeSignedLeb128Checked(&(ptr), begin_ + size_, &(var))) { \ + ErrorStringPrintf("Read out of bounds"); \ + return false; \ } bool DexFileVerifier::CheckAndGetHandlerOffsets(const DexFile::CodeItem* code_item, diff --git a/runtime/entrypoints/quick/quick_alloc_entrypoints.cc b/runtime/entrypoints/quick/quick_alloc_entrypoints.cc index dc5fd07456..515fcbf6e6 100644 --- a/runtime/entrypoints/quick/quick_alloc_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_alloc_entrypoints.cc @@ -43,11 +43,8 @@ extern "C" mirror::Object* artAllocObjectFromCode ##suffix##suffix2( \ obj = self->AllocTlab(byte_count); \ DCHECK(obj != nullptr) << "AllocTlab can't fail"; \ obj->SetClass(klass); \ - if (kUseBakerOrBrooksReadBarrier) { \ - if (kUseBrooksReadBarrier) { \ - obj->SetReadBarrierPointer(obj); \ - } \ - obj->AssertReadBarrierPointer(); \ + if (kUseBakerReadBarrier) { \ + obj->AssertReadBarrierState(); \ } \ QuasiAtomic::ThreadFenceForConstructor(); \ return obj; \ @@ -69,11 +66,8 @@ extern "C" mirror::Object* artAllocObjectFromCodeResolved##suffix##suffix2( \ obj = self->AllocTlab(byte_count); \ DCHECK(obj != nullptr) << "AllocTlab can't fail"; \ obj->SetClass(klass); \ - if (kUseBakerOrBrooksReadBarrier) { \ - if (kUseBrooksReadBarrier) { \ - obj->SetReadBarrierPointer(obj); \ - } \ - obj->AssertReadBarrierPointer(); \ + if (kUseBakerReadBarrier) { \ + obj->AssertReadBarrierState(); \ } \ QuasiAtomic::ThreadFenceForConstructor(); \ return obj; \ @@ -94,11 +88,8 @@ extern "C" mirror::Object* artAllocObjectFromCodeInitialized##suffix##suffix2( \ obj = self->AllocTlab(byte_count); \ DCHECK(obj != nullptr) << "AllocTlab can't fail"; \ obj->SetClass(klass); \ - if (kUseBakerOrBrooksReadBarrier) { \ - if (kUseBrooksReadBarrier) { \ - obj->SetReadBarrierPointer(obj); \ - } \ - obj->AssertReadBarrierPointer(); \ + if (kUseBakerReadBarrier) { \ + obj->AssertReadBarrierState(); \ } \ QuasiAtomic::ThreadFenceForConstructor(); \ return obj; \ diff --git a/runtime/gc/collector/concurrent_copying-inl.h b/runtime/gc/collector/concurrent_copying-inl.h index 76f500c204..7c649525e4 100644 --- a/runtime/gc/collector/concurrent_copying-inl.h +++ b/runtime/gc/collector/concurrent_copying-inl.h @@ -32,7 +32,7 @@ inline mirror::Object* ConcurrentCopying::MarkUnevacFromSpaceRegion( mirror::Object* ref, accounting::ContinuousSpaceBitmap* bitmap) { // For the Baker-style RB, in a rare case, we could incorrectly change the object from white // to gray even though the object has already been marked through. This happens if a mutator - // thread gets preempted before the AtomicSetReadBarrierPointer below, GC marks through the + // thread gets preempted before the AtomicSetReadBarrierState below, GC marks through the // object (changes it from white to gray and back to white), and the thread runs and // incorrectly changes it from white to gray. If this happens, the object will get added to the // mark stack again and get changed back to white after it is processed. @@ -50,14 +50,14 @@ inline mirror::Object* ConcurrentCopying::MarkUnevacFromSpaceRegion( // we can avoid an expensive CAS. // For the baker case, an object is marked if either the mark bit marked or the bitmap bit is // set. - success = ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); + success = ref->AtomicSetReadBarrierState(ReadBarrier::WhiteState(), ReadBarrier::GrayState()); } else { success = !bitmap->AtomicTestAndSet(ref); } if (success) { // Newly marked. if (kUseBakerReadBarrier) { - DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::GrayPtr()); + DCHECK_EQ(ref->GetReadBarrierState(), ReadBarrier::GrayState()); } PushOntoMarkStack(ref); } @@ -84,8 +84,8 @@ inline mirror::Object* ConcurrentCopying::MarkImmuneSpace(mirror::Object* ref) { return ref; } // This may or may not succeed, which is ok because the object may already be gray. - bool success = ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), - ReadBarrier::GrayPtr()); + bool success = ref->AtomicSetReadBarrierState(ReadBarrier::WhiteState(), + ReadBarrier::GrayState()); if (success) { MutexLock mu(Thread::Current(), immune_gray_stack_lock_); immune_gray_stack_.push_back(ref); @@ -125,10 +125,6 @@ inline mirror::Object* ConcurrentCopying::Mark(mirror::Object* from_ref) { return from_ref; case space::RegionSpace::RegionType::kRegionTypeFromSpace: { mirror::Object* to_ref = GetFwdPtr(from_ref); - if (kUseBakerReadBarrier) { - DCHECK_NE(to_ref, ReadBarrier::GrayPtr()) - << "from_ref=" << from_ref << " to_ref=" << to_ref; - } if (to_ref == nullptr) { // It isn't marked yet. Mark it by copying it to the to-space. to_ref = Copy(from_ref); @@ -153,9 +149,7 @@ inline mirror::Object* ConcurrentCopying::Mark(mirror::Object* from_ref) { inline mirror::Object* ConcurrentCopying::MarkFromReadBarrier(mirror::Object* from_ref) { mirror::Object* ret; - // TODO: Delete GetMarkBit check when all of the callers properly check the bit. Remaining caller - // is array allocations. - if (from_ref == nullptr || from_ref->GetMarkBit()) { + if (from_ref == nullptr) { return from_ref; } // TODO: Consider removing this check when we are done investigating slow paths. b/30162165 @@ -192,9 +186,9 @@ inline mirror::Object* ConcurrentCopying::GetFwdPtr(mirror::Object* from_ref) { inline bool ConcurrentCopying::IsMarkedInUnevacFromSpace(mirror::Object* from_ref) { // Use load acquire on the read barrier pointer to ensure that we never see a white read barrier - // pointer with an unmarked bit due to reordering. + // state with an unmarked bit due to reordering. DCHECK(region_space_->IsInUnevacFromSpace(from_ref)); - if (kUseBakerReadBarrier && from_ref->GetReadBarrierPointerAcquire() == ReadBarrier::GrayPtr()) { + if (kUseBakerReadBarrier && from_ref->GetReadBarrierStateAcquire() == ReadBarrier::GrayState()) { return true; } return region_space_bitmap_->Test(from_ref); diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index 6dfab8b566..11d6849681 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -418,7 +418,7 @@ void ConcurrentCopying::VerifyGrayImmuneObjects() { [&visitor](mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) { // If an object is not gray, it should only have references to things in the immune spaces. - if (obj->GetReadBarrierPointer() != ReadBarrier::GrayPtr()) { + if (obj->GetReadBarrierState() != ReadBarrier::GrayState()) { obj->VisitReferences</*kVisitNativeRoots*/true, kDefaultVerifyFlags, kWithoutReadBarrier>(visitor, visitor); @@ -463,7 +463,7 @@ class ConcurrentCopying::GrayImmuneObjectVisitor { if (kIsDebugBuild) { Locks::mutator_lock_->AssertExclusiveHeld(Thread::Current()); } - obj->SetReadBarrierPointer(ReadBarrier::GrayPtr()); + obj->SetReadBarrierState(ReadBarrier::GrayState()); } } @@ -549,11 +549,11 @@ class ConcurrentCopying::ImmuneSpaceScanObjVisitor { ALWAYS_INLINE void operator()(mirror::Object* obj) const REQUIRES_SHARED(Locks::mutator_lock_) { if (kUseBakerReadBarrier && kGrayDirtyImmuneObjects) { - if (obj->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) { + if (obj->GetReadBarrierState() == ReadBarrier::GrayState()) { collector_->ScanImmuneObject(obj); // Done scanning the object, go back to white. - bool success = obj->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(), - ReadBarrier::WhitePtr()); + bool success = obj->AtomicSetReadBarrierState(ReadBarrier::GrayState(), + ReadBarrier::WhiteState()); CHECK(success); } } else { @@ -620,9 +620,9 @@ void ConcurrentCopying::MarkingPhase() { LOG(INFO) << "immune gray stack size=" << immune_gray_stack_.size(); } for (mirror::Object* obj : immune_gray_stack_) { - DCHECK(obj->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); - bool success = obj->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(), - ReadBarrier::WhitePtr()); + DCHECK(obj->GetReadBarrierState() == ReadBarrier::GrayState()); + bool success = obj->AtomicSetReadBarrierState(ReadBarrier::GrayState(), + ReadBarrier::WhiteState()); DCHECK(success); } immune_gray_stack_.clear(); @@ -821,11 +821,11 @@ void ConcurrentCopying::ProcessFalseGrayStack() { for (mirror::Object* obj : false_gray_stack_) { DCHECK(IsMarked(obj)); // The object could be white here if a thread got preempted after a success at the - // AtomicSetReadBarrierPointer in Mark(), GC started marking through it (but not finished so + // AtomicSetReadBarrierState in Mark(), GC started marking through it (but not finished so // still gray), and the thread ran to register it onto the false gray stack. - if (obj->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) { - bool success = obj->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(), - ReadBarrier::WhitePtr()); + if (obj->GetReadBarrierState() == ReadBarrier::GrayState()) { + bool success = obj->AtomicSetReadBarrierState(ReadBarrier::GrayState(), + ReadBarrier::WhiteState()); DCHECK(success); } } @@ -955,9 +955,9 @@ class ConcurrentCopying::VerifyNoFromSpaceRefsVisitor : public SingleRootVisitor } collector_->AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); if (kUseBakerReadBarrier) { - CHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) + CHECK_EQ(ref->GetReadBarrierState(), ReadBarrier::WhiteState()) << "Ref " << ref << " " << ref->PrettyTypeOf() - << " has non-white rb_ptr "; + << " has non-white rb_state "; } } @@ -1026,8 +1026,8 @@ class ConcurrentCopying::VerifyNoFromSpaceRefsObjectVisitor { VerifyNoFromSpaceRefsFieldVisitor visitor(collector); obj->VisitReferences(visitor, visitor); if (kUseBakerReadBarrier) { - CHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) - << "obj=" << obj << " non-white rb_ptr " << obj->GetReadBarrierPointer(); + CHECK_EQ(obj->GetReadBarrierState(), ReadBarrier::WhiteState()) + << "obj=" << obj << " non-white rb_state " << obj->GetReadBarrierState(); } } @@ -1333,8 +1333,8 @@ size_t ConcurrentCopying::ProcessThreadLocalMarkStacks(bool disable_weak_ref_acc inline void ConcurrentCopying::ProcessMarkStackRef(mirror::Object* to_ref) { DCHECK(!region_space_->IsInFromSpace(to_ref)); if (kUseBakerReadBarrier) { - DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) - << " " << to_ref << " " << to_ref->GetReadBarrierPointer() + DCHECK(to_ref->GetReadBarrierState() == ReadBarrier::GrayState()) + << " " << to_ref << " " << to_ref->GetReadBarrierState() << " is_marked=" << IsMarked(to_ref); } bool add_to_live_bytes = false; @@ -1351,8 +1351,8 @@ inline void ConcurrentCopying::ProcessMarkStackRef(mirror::Object* to_ref) { Scan(to_ref); } if (kUseBakerReadBarrier) { - DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) - << " " << to_ref << " " << to_ref->GetReadBarrierPointer() + DCHECK(to_ref->GetReadBarrierState() == ReadBarrier::GrayState()) + << " " << to_ref << " " << to_ref->GetReadBarrierState() << " is_marked=" << IsMarked(to_ref); } #ifdef USE_BAKER_OR_BROOKS_READ_BARRIER @@ -1368,9 +1368,9 @@ inline void ConcurrentCopying::ProcessMarkStackRef(mirror::Object* to_ref) { // above IsInToSpace() evaluates to true and we change the color from gray to white here in this // else block. if (kUseBakerReadBarrier) { - bool success = to_ref->AtomicSetReadBarrierPointer</*kCasRelease*/true>( - ReadBarrier::GrayPtr(), - ReadBarrier::WhitePtr()); + bool success = to_ref->AtomicSetReadBarrierState</*kCasRelease*/true>( + ReadBarrier::GrayState(), + ReadBarrier::WhiteState()); DCHECK(success) << "Must succeed as we won the race."; } } @@ -1458,9 +1458,9 @@ void ConcurrentCopying::CheckEmptyMarkStack() { while (!mark_stack->IsEmpty()) { mirror::Object* obj = mark_stack->PopBack(); if (kUseBakerReadBarrier) { - mirror::Object* rb_ptr = obj->GetReadBarrierPointer(); - LOG(INFO) << "On mark queue : " << obj << " " << obj->PrettyTypeOf() << " rb_ptr=" - << rb_ptr << " is_marked=" << IsMarked(obj); + uint32_t rb_state = obj->GetReadBarrierState(); + LOG(INFO) << "On mark queue : " << obj << " " << obj->PrettyTypeOf() << " rb_state=" + << rb_state << " is_marked=" << IsMarked(obj); } else { LOG(INFO) << "On mark queue : " << obj << " " << obj->PrettyTypeOf() << " is_marked=" << IsMarked(obj); @@ -1707,7 +1707,7 @@ void ConcurrentCopying::AssertToSpaceInvariant(GcRootSource* gc_root_source, void ConcurrentCopying::LogFromSpaceRefHolder(mirror::Object* obj, MemberOffset offset) { if (kUseBakerReadBarrier) { LOG(INFO) << "holder=" << obj << " " << obj->PrettyTypeOf() - << " holder rb_ptr=" << obj->GetReadBarrierPointer(); + << " holder rb_state=" << obj->GetReadBarrierState(); } else { LOG(INFO) << "holder=" << obj << " " << obj->PrettyTypeOf(); } @@ -1762,10 +1762,10 @@ void ConcurrentCopying::AssertToSpaceInvariantInNonMovingSpace(mirror::Object* o return; } bool updated_all_immune_objects = updated_all_immune_objects_.LoadSequentiallyConsistent(); - CHECK(updated_all_immune_objects || ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) - << "Unmarked immune space ref. obj=" << obj << " rb_ptr=" - << (obj != nullptr ? obj->GetReadBarrierPointer() : nullptr) - << " ref=" << ref << " ref rb_ptr=" << ref->GetReadBarrierPointer() + CHECK(updated_all_immune_objects || ref->GetReadBarrierState() == ReadBarrier::GrayState()) + << "Unmarked immune space ref. obj=" << obj << " rb_state=" + << (obj != nullptr ? obj->GetReadBarrierState() : 0U) + << " ref=" << ref << " ref rb_state=" << ref->GetReadBarrierState() << " updated_all_immune_objects=" << updated_all_immune_objects; } } else { @@ -2158,7 +2158,7 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) { to_ref->SetLockWord(old_lock_word, false); // Set the gray ptr. if (kUseBakerReadBarrier) { - to_ref->SetReadBarrierPointer(ReadBarrier::GrayPtr()); + to_ref->SetReadBarrierState(ReadBarrier::GrayState()); } LockWord new_lock_word = LockWord::FromForwardingAddress(reinterpret_cast<size_t>(to_ref)); @@ -2176,7 +2176,7 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) { DCHECK_EQ(bytes_allocated, non_moving_space_bytes_allocated); } if (kUseBakerReadBarrier) { - DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); + DCHECK(to_ref->GetReadBarrierState() == ReadBarrier::GrayState()); } DCHECK(GetFwdPtr(from_ref) == to_ref); CHECK_NE(to_ref->GetLockWord(false).GetState(), LockWord::kForwardingAddress); @@ -2262,14 +2262,14 @@ mirror::Object* ConcurrentCopying::MarkNonMoving(mirror::Object* ref) { if (!is_los && mark_bitmap->Test(ref)) { // Already marked. if (kUseBakerReadBarrier) { - DCHECK(ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || - ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr()); + DCHECK(ref->GetReadBarrierState() == ReadBarrier::GrayState() || + ref->GetReadBarrierState() == ReadBarrier::WhiteState()); } } else if (is_los && los_bitmap->Test(ref)) { // Already marked in LOS. if (kUseBakerReadBarrier) { - DCHECK(ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || - ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr()); + DCHECK(ref->GetReadBarrierState() == ReadBarrier::GrayState() || + ref->GetReadBarrierState() == ReadBarrier::WhiteState()); } } else { // Not marked. @@ -2282,7 +2282,7 @@ mirror::Object* ConcurrentCopying::MarkNonMoving(mirror::Object* ref) { DCHECK(!los_bitmap->Test(ref)); } if (kUseBakerReadBarrier) { - DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()); + DCHECK_EQ(ref->GetReadBarrierState(), ReadBarrier::WhiteState()); } } else { // For the baker-style RB, we need to handle 'false-gray' cases. See the @@ -2298,25 +2298,25 @@ mirror::Object* ConcurrentCopying::MarkNonMoving(mirror::Object* ref) { // This may or may not succeed, which is ok. bool cas_success = false; if (kUseBakerReadBarrier) { - cas_success = ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), - ReadBarrier::GrayPtr()); + cas_success = ref->AtomicSetReadBarrierState(ReadBarrier::WhiteState(), + ReadBarrier::GrayState()); } if (!is_los && mark_bitmap->AtomicTestAndSet(ref)) { // Already marked. if (kUseBakerReadBarrier && cas_success && - ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) { + ref->GetReadBarrierState() == ReadBarrier::GrayState()) { PushOntoFalseGrayStack(ref); } } else if (is_los && los_bitmap->AtomicTestAndSet(ref)) { // Already marked in LOS. if (kUseBakerReadBarrier && cas_success && - ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) { + ref->GetReadBarrierState() == ReadBarrier::GrayState()) { PushOntoFalseGrayStack(ref); } } else { // Newly marked. if (kUseBakerReadBarrier) { - DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::GrayPtr()); + DCHECK_EQ(ref->GetReadBarrierState(), ReadBarrier::GrayState()); } PushOntoMarkStack(ref); } diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc index e0bf744f4c..ddcb6c0698 100644 --- a/runtime/gc/collector/mark_compact.cc +++ b/runtime/gc/collector/mark_compact.cc @@ -124,9 +124,9 @@ inline mirror::Object* MarkCompact::MarkObject(mirror::Object* obj) { if (obj == nullptr) { return nullptr; } - if (kUseBakerOrBrooksReadBarrier) { - // Verify all the objects have the correct forward pointer installed. - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + // Verify all the objects have the correct forward state installed. + obj->AssertReadBarrierState(); } if (!immune_spaces_.IsInImmuneRegion(obj)) { if (objects_before_forwarding_->HasAddress(obj)) { diff --git a/runtime/gc/collector/mark_sweep.cc b/runtime/gc/collector/mark_sweep.cc index 77d7274a53..7b73e43ad2 100644 --- a/runtime/gc/collector/mark_sweep.cc +++ b/runtime/gc/collector/mark_sweep.cc @@ -462,9 +462,9 @@ inline void MarkSweep::MarkObjectNonNull(mirror::Object* obj, mirror::Object* holder, MemberOffset offset) { DCHECK(obj != nullptr); - if (kUseBakerOrBrooksReadBarrier) { - // Verify all the objects have the correct pointer installed. - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + // Verify all the objects have the correct state installed. + obj->AssertReadBarrierState(); } if (immune_spaces_.IsInImmuneRegion(obj)) { if (kCountMarkedObjects) { @@ -503,9 +503,9 @@ inline void MarkSweep::PushOnMarkStack(mirror::Object* obj) { inline bool MarkSweep::MarkObjectParallel(mirror::Object* obj) { DCHECK(obj != nullptr); - if (kUseBakerOrBrooksReadBarrier) { - // Verify all the objects have the correct pointer installed. - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + // Verify all the objects have the correct state installed. + obj->AssertReadBarrierState(); } if (immune_spaces_.IsInImmuneRegion(obj)) { DCHECK(IsMarked(obj) != nullptr); diff --git a/runtime/gc/collector/semi_space.cc b/runtime/gc/collector/semi_space.cc index 2ff4a3f57a..a815b830c1 100644 --- a/runtime/gc/collector/semi_space.cc +++ b/runtime/gc/collector/semi_space.cc @@ -589,13 +589,9 @@ mirror::Object* SemiSpace::MarkNonForwardedObject(mirror::Object* obj) { // references. saved_bytes_ += CopyAvoidingDirtyingPages(reinterpret_cast<void*>(forward_address), obj, object_size); - if (kUseBakerOrBrooksReadBarrier) { - obj->AssertReadBarrierPointer(); - if (kUseBrooksReadBarrier) { - DCHECK_EQ(forward_address->GetReadBarrierPointer(), obj); - forward_address->SetReadBarrierPointer(forward_address); - } - forward_address->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); + forward_address->AssertReadBarrierState(); } DCHECK(to_space_->HasAddress(forward_address) || fallback_space_->HasAddress(forward_address) || diff --git a/runtime/gc/heap-inl.h b/runtime/gc/heap-inl.h index 05ce9c7952..97129e8b19 100644 --- a/runtime/gc/heap-inl.h +++ b/runtime/gc/heap-inl.h @@ -86,11 +86,8 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, obj = self->AllocTlab(byte_count); DCHECK(obj != nullptr) << "AllocTlab can't fail"; obj->SetClass(klass); - if (kUseBakerOrBrooksReadBarrier) { - if (kUseBrooksReadBarrier) { - obj->SetReadBarrierPointer(obj.Ptr()); - } - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); } bytes_allocated = byte_count; usable_size = bytes_allocated; @@ -102,11 +99,8 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, LIKELY(obj != nullptr)) { DCHECK(!is_running_on_memory_tool_); obj->SetClass(klass); - if (kUseBakerOrBrooksReadBarrier) { - if (kUseBrooksReadBarrier) { - obj->SetReadBarrierPointer(obj.Ptr()); - } - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); } usable_size = bytes_allocated; pre_fence_visitor(obj, usable_size); @@ -143,11 +137,8 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, DCHECK_GT(bytes_allocated, 0u); DCHECK_GT(usable_size, 0u); obj->SetClass(klass); - if (kUseBakerOrBrooksReadBarrier) { - if (kUseBrooksReadBarrier) { - obj->SetReadBarrierPointer(obj.Ptr()); - } - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); } if (collector::SemiSpace::kUseRememberedSet && UNLIKELY(allocator == kAllocatorTypeNonMoving)) { // (Note this if statement will be constant folded away for the diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index 5de004b7a3..19760afed7 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -2406,13 +2406,9 @@ class ZygoteCompactingCollector FINAL : public collector::SemiSpace { } // Copy the object over to its new location. Don't use alloc_size to avoid valgrind error. memcpy(reinterpret_cast<void*>(forward_address), obj, obj_size); - if (kUseBakerOrBrooksReadBarrier) { - obj->AssertReadBarrierPointer(); - if (kUseBrooksReadBarrier) { - DCHECK_EQ(forward_address->GetReadBarrierPointer(), obj); - forward_address->SetReadBarrierPointer(forward_address); - } - forward_address->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); + forward_address->AssertReadBarrierState(); } return forward_address; } diff --git a/runtime/gc/reference_queue.cc b/runtime/gc/reference_queue.cc index 4e6f7da5f0..a0eb197bd5 100644 --- a/runtime/gc/reference_queue.cc +++ b/runtime/gc/reference_queue.cc @@ -75,19 +75,19 @@ ObjPtr<mirror::Reference> ReferenceQueue::DequeuePendingReference() { // collector (SemiSpace) is running. CHECK(ref != nullptr); collector::ConcurrentCopying* concurrent_copying = heap->ConcurrentCopyingCollector(); - mirror::Object* rb_ptr = ref->GetReadBarrierPointer(); - if (rb_ptr == ReadBarrier::GrayPtr()) { - ref->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(), ReadBarrier::WhitePtr()); - CHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()); + uint32_t rb_state = ref->GetReadBarrierState(); + if (rb_state == ReadBarrier::GrayState()) { + ref->AtomicSetReadBarrierState(ReadBarrier::GrayState(), ReadBarrier::WhiteState()); + CHECK_EQ(ref->GetReadBarrierState(), ReadBarrier::WhiteState()); } else { // In ConcurrentCopying::ProcessMarkStackRef() we may leave a white reference in the queue and // find it here, which is OK. - CHECK_EQ(rb_ptr, ReadBarrier::WhitePtr()) << "ref=" << ref << " rb_ptr=" << rb_ptr; + CHECK_EQ(rb_state, ReadBarrier::WhiteState()) << "ref=" << ref << " rb_state=" << rb_state; ObjPtr<mirror::Object> referent = ref->GetReferent<kWithoutReadBarrier>(); // The referent could be null if it's cleared by a mutator (Reference.clear()). if (referent != nullptr) { CHECK(concurrent_copying->IsInToSpace(referent.Ptr())) - << "ref=" << ref << " rb_ptr=" << ref->GetReadBarrierPointer() + << "ref=" << ref << " rb_state=" << ref->GetReadBarrierState() << " referent=" << referent; } } diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index 6035406dd0..0b602e946a 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -399,8 +399,8 @@ void ImageSpace::VerifyImageAllocations() { auto* obj = reinterpret_cast<mirror::Object*>(current); CHECK(obj->GetClass() != nullptr) << "Image object at address " << obj << " has null class"; CHECK(live_bitmap_->Test(obj)) << obj->PrettyTypeOf(); - if (kUseBakerOrBrooksReadBarrier) { - obj->AssertReadBarrierPointer(); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); } current += RoundUp(obj->SizeOf(), kObjectAlignment); } @@ -1606,7 +1606,7 @@ std::unique_ptr<ImageSpace> ImageSpace::CreateBootImage(const char* image_locati std::ostringstream oss; bool first = true; - for (auto msg : error_msgs) { + for (const auto& msg : error_msgs) { if (!first) { oss << "\n "; } diff --git a/runtime/gc/space/space_test.h b/runtime/gc/space/space_test.h index 7778871060..cbb3d73497 100644 --- a/runtime/gc/space/space_test.h +++ b/runtime/gc/space/space_test.h @@ -108,13 +108,10 @@ class SpaceTest : public Super { EXPECT_GE(size, SizeOfZeroLengthByteArray()); EXPECT_TRUE(byte_array_class != nullptr); o->SetClass(byte_array_class); - if (kUseBakerOrBrooksReadBarrier) { + if (kUseBakerReadBarrier) { // Like the proper heap object allocation, install and verify - // the correct read barrier pointer. - if (kUseBrooksReadBarrier) { - o->SetReadBarrierPointer(o); - } - o->AssertReadBarrierPointer(); + // the correct read barrier state. + o->AssertReadBarrierState(); } mirror::Array* arr = o->AsArray<kVerifyNone>(); size_t header_size = SizeOfZeroLengthByteArray(); diff --git a/runtime/gc/system_weak.h b/runtime/gc/system_weak.h index 3910a280cc..887059b57f 100644 --- a/runtime/gc/system_weak.h +++ b/runtime/gc/system_weak.h @@ -69,6 +69,11 @@ class SystemWeakHolder : public AbstractSystemWeakHolder { new_weak_condition_.Broadcast(Thread::Current()); } + // WARNING: For lock annotations only. + Mutex* GetAllowDisallowLock() const RETURN_CAPABILITY(allow_disallow_lock_) { + return nullptr; + } + protected: void Wait(Thread* self) REQUIRES_SHARED(allow_disallow_lock_) { // Wait for GC's sweeping to complete and allow new records diff --git a/runtime/generate-operator-out.py b/runtime/generate-operator-out.py new file mode 120000 index 0000000000..cc291d20c1 --- /dev/null +++ b/runtime/generate-operator-out.py @@ -0,0 +1 @@ +../tools/generate-operator-out.py
\ No newline at end of file diff --git a/runtime/generated/asm_support_gen.h b/runtime/generated/asm_support_gen.h index 6c189b0a84..2c95fe942e 100644 --- a/runtime/generated/asm_support_gen.h +++ b/runtime/generated/asm_support_gen.h @@ -98,6 +98,12 @@ DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_READ_BARRIER_STATE_MASK), (stati DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), (static_cast<uint32_t>(art::LockWord::kReadBarrierStateMaskShiftedToggled))) #define LOCK_WORD_THIN_LOCK_COUNT_ONE 65536 DEFINE_CHECK_EQ(static_cast<int32_t>(LOCK_WORD_THIN_LOCK_COUNT_ONE), (static_cast<int32_t>(art::LockWord::kThinLockCountOne))) +#define LOCK_WORD_STATE_FORWARDING_ADDRESS 0x3 +DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_STATE_FORWARDING_ADDRESS), (static_cast<uint32_t>(art::LockWord::kStateForwardingAddress))) +#define LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW 0x40000000 +DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW), (static_cast<uint32_t>(art::LockWord::kStateForwardingAddressOverflow))) +#define LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 0x3 +DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT), (static_cast<uint32_t>(art::LockWord::kForwardingAddressShift))) #define LOCK_WORD_GC_STATE_MASK_SHIFTED 0x30000000 DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_GC_STATE_MASK_SHIFTED), (static_cast<uint32_t>(art::LockWord::kGCStateMaskShifted))) #define LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED 0xcfffffff diff --git a/runtime/handle.h b/runtime/handle.h index d33d4a638a..3db3be202a 100644 --- a/runtime/handle.h +++ b/runtime/handle.h @@ -42,13 +42,9 @@ class Handle : public ValueObject { Handle() : reference_(nullptr) { } - ALWAYS_INLINE Handle(const Handle<T>& handle) : reference_(handle.reference_) { - } + ALWAYS_INLINE Handle(const Handle<T>& handle) = default; - ALWAYS_INLINE Handle<T>& operator=(const Handle<T>& handle) { - reference_ = handle.reference_; - return *this; - } + ALWAYS_INLINE Handle<T>& operator=(const Handle<T>& handle) = default; ALWAYS_INLINE explicit Handle(StackReference<T>* reference) : reference_(reference) { } @@ -109,15 +105,10 @@ class MutableHandle : public Handle<T> { } ALWAYS_INLINE MutableHandle(const MutableHandle<T>& handle) - REQUIRES_SHARED(Locks::mutator_lock_) - : Handle<T>(handle.reference_) { - } + REQUIRES_SHARED(Locks::mutator_lock_) = default; ALWAYS_INLINE MutableHandle<T>& operator=(const MutableHandle<T>& handle) - REQUIRES_SHARED(Locks::mutator_lock_) { - Handle<T>::operator=(handle); - return *this; - } + REQUIRES_SHARED(Locks::mutator_lock_) = default; ALWAYS_INLINE explicit MutableHandle(StackReference<T>* reference) REQUIRES_SHARED(Locks::mutator_lock_) diff --git a/runtime/handle_scope.h b/runtime/handle_scope.h index 8a0aba6121..adb7d8a10c 100644 --- a/runtime/handle_scope.h +++ b/runtime/handle_scope.h @@ -69,7 +69,7 @@ class PACKED(4) BaseHandleScope { number_of_references_(num_references) {} // Variable sized constructor. - BaseHandleScope(BaseHandleScope* link) + explicit BaseHandleScope(BaseHandleScope* link) : link_(link), number_of_references_(kNumReferencesVariableSized) {} diff --git a/runtime/handle_scope_test.cc b/runtime/handle_scope_test.cc index 92063c4ba8..aab1d9c224 100644 --- a/runtime/handle_scope_test.cc +++ b/runtime/handle_scope_test.cc @@ -14,15 +14,27 @@ * limitations under the License. */ +#include <type_traits> + #include "base/enums.h" #include "common_runtime_test.h" #include "gtest/gtest.h" +#include "handle.h" #include "handle_scope-inl.h" +#include "mirror/object.h" #include "scoped_thread_state_change-inl.h" #include "thread.h" namespace art { +// Handles are value objects and should be trivially copyable. +static_assert(std::is_trivially_copyable<Handle<mirror::Object>>::value, + "Handle should be trivially copyable"); +static_assert(std::is_trivially_copyable<MutableHandle<mirror::Object>>::value, + "MutableHandle should be trivially copyable"); +static_assert(std::is_trivially_copyable<ScopedNullHandle<mirror::Object>>::value, + "ScopedNullHandle should be trivially copyable"); + class HandleScopeTest : public CommonRuntimeTest {}; // Test the offsets computed for members of HandleScope. Because of cross-compiling diff --git a/runtime/hprof/hprof.cc b/runtime/hprof/hprof.cc index 233675980d..8cbe49166a 100644 --- a/runtime/hprof/hprof.cc +++ b/runtime/hprof/hprof.cc @@ -574,9 +574,9 @@ class Hprof : public SingleRootVisitor { } void WriteStringTable() { - for (const std::pair<std::string, HprofStringId>& p : strings_) { + for (const auto& p : strings_) { const std::string& string = p.first; - const size_t id = p.second; + const HprofStringId id = p.second; output_->StartNewRecord(HPROF_TAG_STRING, kHprofTime); diff --git a/runtime/intern_table.cc b/runtime/intern_table.cc index 7b7510979d..d885226bb0 100644 --- a/runtime/intern_table.cc +++ b/runtime/intern_table.cc @@ -199,7 +199,8 @@ void InternTable::WaitUntilAccessible(Thread* self) { { ScopedThreadSuspension sts(self, kWaitingWeakGcRootRead); MutexLock mu(self, *Locks::intern_table_lock_); - while (weak_root_state_ == gc::kWeakRootStateNoReadsOrWrites) { + while ((!kUseReadBarrier && weak_root_state_ == gc::kWeakRootStateNoReadsOrWrites) || + (kUseReadBarrier && !self->GetWeakRefAccessEnabled())) { weak_intern_condition_.Wait(self); } } diff --git a/runtime/interpreter/interpreter_common.cc b/runtime/interpreter/interpreter_common.cc index 1a99100eb4..4843c4dc59 100644 --- a/runtime/interpreter/interpreter_common.cc +++ b/runtime/interpreter/interpreter_common.cc @@ -48,7 +48,8 @@ static ALWAYS_INLINE void DoFieldGetCommon(Thread* self, const ShadowFrame& shadow_frame, ObjPtr<mirror::Object>& obj, ArtField* field, - JValue* result) REQUIRES_SHARED(Locks::mutator_lock_) { + JValue* result) + REQUIRES_SHARED(Locks::mutator_lock_) { field->GetDeclaringClass()->AssertInitializedOrInitializingInThread(self); // Report this field access to instrumentation if needed. @@ -299,6 +300,42 @@ EXPLICIT_DO_IGET_QUICK_TEMPLATE_DECL(Primitive::kPrimLong); // iget-wide-qui EXPLICIT_DO_IGET_QUICK_TEMPLATE_DECL(Primitive::kPrimNot); // iget-object-quick. #undef EXPLICIT_DO_IGET_QUICK_TEMPLATE_DECL +static JValue GetFieldValue(const ShadowFrame& shadow_frame, + Primitive::Type field_type, + uint32_t vreg) + REQUIRES_SHARED(Locks::mutator_lock_) { + JValue field_value; + switch (field_type) { + case Primitive::kPrimBoolean: + field_value.SetZ(static_cast<uint8_t>(shadow_frame.GetVReg(vreg))); + break; + case Primitive::kPrimByte: + field_value.SetB(static_cast<int8_t>(shadow_frame.GetVReg(vreg))); + break; + case Primitive::kPrimChar: + field_value.SetC(static_cast<uint16_t>(shadow_frame.GetVReg(vreg))); + break; + case Primitive::kPrimShort: + field_value.SetS(static_cast<int16_t>(shadow_frame.GetVReg(vreg))); + break; + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + field_value.SetI(shadow_frame.GetVReg(vreg)); + break; + case Primitive::kPrimLong: + case Primitive::kPrimDouble: + field_value.SetJ(shadow_frame.GetVRegLong(vreg)); + break; + case Primitive::kPrimNot: + field_value.SetL(shadow_frame.GetVRegReference(vreg)); + break; + case Primitive::kPrimVoid: + LOG(FATAL) << "Unreachable: " << field_type; + UNREACHABLE(); + } + return field_value; +} + template<Primitive::Type field_type> static JValue GetFieldValue(const ShadowFrame& shadow_frame, uint32_t vreg) REQUIRES_SHARED(Locks::mutator_lock_) { @@ -337,7 +374,8 @@ static inline bool DoFieldPutCommon(Thread* self, const ShadowFrame& shadow_frame, ObjPtr<mirror::Object>& obj, ArtField* f, - size_t vregA) REQUIRES_SHARED(Locks::mutator_lock_) { + const JValue& value) + REQUIRES_SHARED(Locks::mutator_lock_) { f->GetDeclaringClass()->AssertInitializedOrInitializingInThread(self); // Report this field access to instrumentation if needed. Since we only have the offset of @@ -347,36 +385,35 @@ static inline bool DoFieldPutCommon(Thread* self, StackHandleScope<1> hs(self); // Wrap in handle wrapper in case the listener does thread suspension. HandleWrapperObjPtr<mirror::Object> h(hs.NewHandleWrapper(&obj)); - JValue field_value = GetFieldValue<field_type>(shadow_frame, vregA); ObjPtr<mirror::Object> this_object = f->IsStatic() ? nullptr : obj; instrumentation->FieldWriteEvent(self, this_object.Ptr(), shadow_frame.GetMethod(), shadow_frame.GetDexPC(), f, - field_value); + value); } switch (field_type) { case Primitive::kPrimBoolean: - f->SetBoolean<transaction_active>(obj, shadow_frame.GetVReg(vregA)); + f->SetBoolean<transaction_active>(obj, value.GetZ()); break; case Primitive::kPrimByte: - f->SetByte<transaction_active>(obj, shadow_frame.GetVReg(vregA)); + f->SetByte<transaction_active>(obj, value.GetB()); break; case Primitive::kPrimChar: - f->SetChar<transaction_active>(obj, shadow_frame.GetVReg(vregA)); + f->SetChar<transaction_active>(obj, value.GetC()); break; case Primitive::kPrimShort: - f->SetShort<transaction_active>(obj, shadow_frame.GetVReg(vregA)); + f->SetShort<transaction_active>(obj, value.GetS()); break; case Primitive::kPrimInt: - f->SetInt<transaction_active>(obj, shadow_frame.GetVReg(vregA)); + f->SetInt<transaction_active>(obj, value.GetI()); break; case Primitive::kPrimLong: - f->SetLong<transaction_active>(obj, shadow_frame.GetVRegLong(vregA)); + f->SetLong<transaction_active>(obj, value.GetJ()); break; case Primitive::kPrimNot: { - ObjPtr<mirror::Object> reg = shadow_frame.GetVRegReference(vregA); + ObjPtr<mirror::Object> reg = value.GetL(); if (do_assignability_check && reg != nullptr) { // FieldHelper::GetType can resolve classes, use a handle wrapper which will restore the // object in the destructor. @@ -434,11 +471,12 @@ bool DoFieldPut(Thread* self, const ShadowFrame& shadow_frame, const Instruction } uint32_t vregA = is_static ? inst->VRegA_21c(inst_data) : inst->VRegA_22c(inst_data); + JValue value = GetFieldValue<field_type>(shadow_frame, vregA); return DoFieldPutCommon<field_type, do_assignability_check, transaction_active>(self, shadow_frame, obj, f, - vregA); + value); } // Explicitly instantiate all DoFieldPut functions. @@ -479,37 +517,34 @@ bool DoFieldPutForInvokePolymorphic(Thread* self, ObjPtr<mirror::Object>& obj, ArtField* field, Primitive::Type field_type, - size_t vregA) REQUIRES_SHARED(Locks::mutator_lock_) { + const JValue& value) + REQUIRES_SHARED(Locks::mutator_lock_) { static const bool kDoCheckAssignability = false; static const bool kTransaction = false; switch (field_type) { case Primitive::kPrimBoolean: return DoFieldPutCommon<Primitive::kPrimBoolean, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); + self, shadow_frame, obj, field, value); case Primitive::kPrimByte: return DoFieldPutCommon<Primitive::kPrimByte, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); + self, shadow_frame, obj, field, value); case Primitive::kPrimChar: return DoFieldPutCommon<Primitive::kPrimChar, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); + self, shadow_frame, obj, field, value); case Primitive::kPrimShort: return DoFieldPutCommon<Primitive::kPrimShort, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); + self, shadow_frame, obj, field, value); case Primitive::kPrimInt: - return DoFieldPutCommon<Primitive::kPrimInt, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); - case Primitive::kPrimLong: - return DoFieldPutCommon<Primitive::kPrimLong, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); case Primitive::kPrimFloat: return DoFieldPutCommon<Primitive::kPrimInt, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); + self, shadow_frame, obj, field, value); + case Primitive::kPrimLong: case Primitive::kPrimDouble: return DoFieldPutCommon<Primitive::kPrimLong, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); + self, shadow_frame, obj, field, value); case Primitive::kPrimNot: return DoFieldPutCommon<Primitive::kPrimNot, kDoCheckAssignability, kTransaction>( - self, shadow_frame, obj, field, vregA); + self, shadow_frame, obj, field, value); case Primitive::kPrimVoid: LOG(FATAL) << "Unreachable: " << field_type; UNREACHABLE(); @@ -774,12 +809,31 @@ inline static bool IsInvokeExact(const DexFile& dex_file, int invoke_method_idx) return is_invoke_exact; } +inline static ObjPtr<mirror::Class> GetAndInitializeDeclaringClass(Thread* self, ArtField* field) + REQUIRES_SHARED(Locks::mutator_lock_) { + // Method handle invocations on static fields should ensure class is + // initialized. This usually happens when an instance is constructed + // or class members referenced, but this is not guaranteed when + // looking up method handles. + ObjPtr<mirror::Class> klass = field->GetDeclaringClass(); + if (UNLIKELY(!klass->IsInitialized())) { + StackHandleScope<1> hs(self); + HandleWrapperObjPtr<mirror::Class> h(hs.NewHandleWrapper(&klass)); + if (!Runtime::Current()->GetClassLinker()->EnsureInitialized(self, h, true, true)) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } + } + return klass; +} + template<bool is_range, bool do_access_check> inline bool DoInvokePolymorphic(Thread* self, ShadowFrame& shadow_frame, const Instruction* inst, uint16_t inst_data, - JValue* result) REQUIRES_SHARED(Locks::mutator_lock_) { + JValue* result) + REQUIRES_SHARED(Locks::mutator_lock_) { // Invoke-polymorphic instructions always take a receiver. i.e, they are never static. const uint32_t vRegC = (is_range) ? inst->VRegC_4rcc() : inst->VRegC_45cc(); const int invoke_method_idx = (is_range) ? inst->VRegB_4rcc() : inst->VRegB_45cc(); @@ -833,9 +887,23 @@ inline bool DoInvokePolymorphic(Thread* self, const MethodHandleKind handle_kind = method_handle->GetHandleKind(); Handle<mirror::MethodType> handle_type(hs.NewHandle(method_handle->GetMethodType())); CHECK(handle_type.Get() != nullptr); - if (UNLIKELY(is_invoke_exact && !callsite_type->IsExactMatch(handle_type.Get()))) { - ThrowWrongMethodTypeException(handle_type.Get(), callsite_type.Get()); - return false; + if (is_invoke_exact) { + // We need to check the nominal type of the handle in addition to the + // real type. The "nominal" type is present when MethodHandle.asType is + // called any handle, and results in the declared type of the handle + // changing. + ObjPtr<mirror::MethodType> nominal_type(method_handle->GetNominalType()); + ObjPtr<mirror::MethodType> check_type(nullptr); + if (LIKELY(nominal_type.Ptr() == nullptr)) { + check_type.Assign(handle_type.Get()); + } else { + check_type.Assign(nominal_type.Ptr()); + } + + if (UNLIKELY(!callsite_type->IsExactMatch(check_type.Ptr()))) { + ThrowWrongMethodTypeException(check_type.Ptr(), callsite_type.Get()); + return false; + } } uint32_t arg[Instruction::kMaxVarArgRegs] = {}; @@ -928,28 +996,55 @@ inline bool DoInvokePolymorphic(Thread* self, Primitive::Type field_type = field->GetTypeAsPrimitiveType();; if (!is_invoke_exact) { - // TODO(oth): conversion plumbing for invoke(). - UNIMPLEMENTED(FATAL); + if (handle_type->GetPTypes()->GetLength() != callsite_type->GetPTypes()->GetLength()) { + // Too many arguments to setter or getter. + ThrowWrongMethodTypeException(callsite_type.Get(), handle_type.Get()); + return false; + } } switch (handle_kind) { case kInstanceGet: { ObjPtr<mirror::Object> obj = shadow_frame.GetVRegReference(first_src_reg); DoFieldGetForInvokePolymorphic(self, shadow_frame, obj, field, field_type, result); + if (!ConvertReturnValue(callsite_type, handle_type, result)) { + DCHECK(self->IsExceptionPending()); + return false; + } return true; } - case kInstancePut: { - ObjPtr<mirror::Object> obj = shadow_frame.GetVRegReference(first_src_reg); - return DoFieldPutForInvokePolymorphic(self, shadow_frame, obj, field, field_type, arg[1]); - } case kStaticGet: { - ObjPtr<mirror::Object> obj = field->GetDeclaringClass(); + ObjPtr<mirror::Object> obj = GetAndInitializeDeclaringClass(self, field); + if (obj == nullptr) { + DCHECK(self->IsExceptionPending()); + return false; + } DoFieldGetForInvokePolymorphic(self, shadow_frame, obj, field, field_type, result); + if (!ConvertReturnValue(callsite_type, handle_type, result)) { + DCHECK(self->IsExceptionPending()); + return false; + } return true; } + case kInstancePut: { + JValue value = GetFieldValue(shadow_frame, field_type, arg[1]); + if (!ConvertArgumentValue(callsite_type, handle_type, 1, &value)) { + DCHECK(self->IsExceptionPending()); + return false; + } + ObjPtr<mirror::Object> obj = shadow_frame.GetVRegReference(first_src_reg); + result->SetL(0); + return DoFieldPutForInvokePolymorphic(self, shadow_frame, obj, field, field_type, value); + } case kStaticPut: { + JValue value = GetFieldValue(shadow_frame, field_type, arg[0]); + if (!ConvertArgumentValue(callsite_type, handle_type, 0, &value)) { + DCHECK(self->IsExceptionPending()); + return false; + } ObjPtr<mirror::Object> obj = field->GetDeclaringClass(); - return DoFieldPutForInvokePolymorphic(self, shadow_frame, obj, field, field_type, arg[0]); + result->SetL(0); + return DoFieldPutForInvokePolymorphic(self, shadow_frame, obj, field, field_type, value); } default: LOG(FATAL) << "Unreachable: " << handle_kind; diff --git a/runtime/jdwp/jdwp_event.cc b/runtime/jdwp/jdwp_event.cc index 85bfd17ef5..fad7d90ab8 100644 --- a/runtime/jdwp/jdwp_event.cc +++ b/runtime/jdwp/jdwp_event.cc @@ -781,7 +781,7 @@ void JdwpState::PostVMStart() { SendRequestAndPossiblySuspend(pReq, suspend_policy, threadId); } -static void LogMatchingEventsAndThread(const std::vector<JdwpEvent*> match_list, +static void LogMatchingEventsAndThread(const std::vector<JdwpEvent*>& match_list, ObjectId thread_id) REQUIRES_SHARED(Locks::mutator_lock_) { for (size_t i = 0, e = match_list.size(); i < e; ++i) { diff --git a/runtime/lock_word.h b/runtime/lock_word.h index 538b6ebbba..2f2565b9d0 100644 --- a/runtime/lock_word.h +++ b/runtime/lock_word.h @@ -61,7 +61,7 @@ class Monitor; */ class LockWord { public: - enum SizeShiftsAndMasks { // private marker to avoid generate-operator-out.py from processing. + enum SizeShiftsAndMasks : uint32_t { // private marker to avoid generate-operator-out.py from processing. // Number of bits to encode the state, currently just fat or thin/unlocked or hash code. kStateSize = 2, kReadBarrierStateSize = 1, @@ -91,6 +91,8 @@ class LockWord { kStateFat = 1, kStateHash = 2, kStateForwardingAddress = 3, + kStateForwardingAddressShifted = kStateForwardingAddress << kStateShift, + kStateForwardingAddressOverflow = (1 + kStateMask - kStateForwardingAddress) << kStateShift, // Read barrier bit. kReadBarrierStateShift = kThinLockCountSize + kThinLockCountShift, @@ -140,7 +142,7 @@ class LockWord { static LockWord FromForwardingAddress(size_t target) { DCHECK_ALIGNED(target, (1 << kStateSize)); - return LockWord((target >> kForwardingAddressShift) | (kStateForwardingAddress << kStateShift)); + return LockWord((target >> kForwardingAddressShift) | kStateForwardingAddressShifted); } static LockWord FromHashCode(uint32_t hash_code, uint32_t gc_state) { @@ -202,6 +204,8 @@ class LockWord { void SetReadBarrierState(uint32_t rb_state) { DCHECK_EQ(rb_state & ~kReadBarrierStateMask, 0U); + DCHECK(rb_state == ReadBarrier::WhiteState() || + rb_state == ReadBarrier::GrayState()) << rb_state; DCHECK_NE(static_cast<uint32_t>(GetState()), static_cast<uint32_t>(kForwardingAddress)); // Clear and or the bits. value_ &= ~(kReadBarrierStateMask << kReadBarrierStateShift); @@ -256,6 +260,14 @@ class LockWord { LockWord(); explicit LockWord(uint32_t val) : value_(val) { + // Make sure adding the overflow causes an overflow. + constexpr uint64_t overflow = static_cast<uint64_t>(kStateForwardingAddressShifted) + + static_cast<uint64_t>(kStateForwardingAddressOverflow); + constexpr bool is_larger = overflow > static_cast<uint64_t>(0xFFFFFFFF); + static_assert(is_larger, "should have overflowed"); + static_assert( + (~kStateForwardingAddress & kStateMask) == 0, + "READ_BARRIER_MARK_REG relies on the forwarding address state being only one bits"); CheckReadBarrierState(); } @@ -270,9 +282,8 @@ class LockWord { if (!kUseReadBarrier) { DCHECK_EQ(rb_state, 0U); } else { - DCHECK(rb_state == ReadBarrier::white_ptr_ || - rb_state == ReadBarrier::gray_ptr_ || - rb_state == ReadBarrier::black_ptr_) << rb_state; + DCHECK(rb_state == ReadBarrier::WhiteState() || + rb_state == ReadBarrier::GrayState()) << rb_state; } } } diff --git a/runtime/method_handles-inl.h b/runtime/method_handles-inl.h index b279a3778c..1240792643 100644 --- a/runtime/method_handles-inl.h +++ b/runtime/method_handles-inl.h @@ -31,127 +31,70 @@ namespace art { -// Assigns |type| to the primitive type associated with |dst_class|. Returns -// true iff. |dst_class| was a boxed type (Integer, Long etc.), false otherwise. -REQUIRES_SHARED(Locks::mutator_lock_) -static inline bool GetPrimitiveType(ObjPtr<mirror::Class> dst_class, Primitive::Type* type) { - if (dst_class->DescriptorEquals("Ljava/lang/Boolean;")) { - (*type) = Primitive::kPrimBoolean; +inline bool ConvertArgumentValue(Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, + int index, + JValue* value) REQUIRES_SHARED(Locks::mutator_lock_) { + ObjPtr<mirror::Class> from_class(callsite_type->GetPTypes()->GetWithoutChecks(index)); + ObjPtr<mirror::Class> to_class(callee_type->GetPTypes()->GetWithoutChecks(index)); + if (from_class == to_class) { return true; - } else if (dst_class->DescriptorEquals("Ljava/lang/Byte;")) { - (*type) = Primitive::kPrimByte; - return true; - } else if (dst_class->DescriptorEquals("Ljava/lang/Character;")) { - (*type) = Primitive::kPrimChar; - return true; - } else if (dst_class->DescriptorEquals("Ljava/lang/Float;")) { - (*type) = Primitive::kPrimFloat; - return true; - } else if (dst_class->DescriptorEquals("Ljava/lang/Double;")) { - (*type) = Primitive::kPrimDouble; - return true; - } else if (dst_class->DescriptorEquals("Ljava/lang/Integer;")) { - (*type) = Primitive::kPrimInt; - return true; - } else if (dst_class->DescriptorEquals("Ljava/lang/Long;")) { - (*type) = Primitive::kPrimLong; - return true; - } else if (dst_class->DescriptorEquals("Ljava/lang/Short;")) { - (*type) = Primitive::kPrimShort; + } + + // |value| may contain a bare heap pointer which is generally + // |unsafe. ConvertJValueCommon() saves |value|, |from_class|, and + // |to_class| to Handles where necessary to avoid issues if the heap + // changes. + if (ConvertJValueCommon(callsite_type, callee_type, from_class, to_class, value)) { + DCHECK(!Thread::Current()->IsExceptionPending()); return true; } else { + DCHECK(Thread::Current()->IsExceptionPending()); + value->SetJ(0); return false; } } -inline bool ConvertJValue(Handle<mirror::Class> from, - Handle<mirror::Class> to, - const JValue& from_value, - JValue* to_value) { - const Primitive::Type from_type = from->GetPrimitiveType(); - const Primitive::Type to_type = to->GetPrimitiveType(); - - // This method must be called only when the types don't match. - DCHECK(from.Get() != to.Get()); - - if ((from_type != Primitive::kPrimNot) && (to_type != Primitive::kPrimNot)) { - // Throws a ClassCastException if we're unable to convert a primitive value. - return ConvertPrimitiveValue(false, from_type, to_type, from_value, to_value); - } else if ((from_type == Primitive::kPrimNot) && (to_type == Primitive::kPrimNot)) { - // They're both reference types. If "from" is null, we can pass it - // through unchanged. If not, we must generate a cast exception if - // |to| is not assignable from the dynamic type of |ref|. - mirror::Object* const ref = from_value.GetL(); - if (ref == nullptr || to->IsAssignableFrom(ref->GetClass())) { - to_value->SetL(ref); - return true; - } else { - ThrowClassCastException(to.Get(), ref->GetClass()); - return false; - } - } else { - // Precisely one of the source or the destination are reference types. - // We must box or unbox. - if (to_type == Primitive::kPrimNot) { - // The target type is a reference, we must box. - Primitive::Type type; - // TODO(narayan): This is a CHECK for now. There might be a few corner cases - // here that we might not have handled yet. For exmple, if |to| is java/lang/Number;, - // we will need to box this "naturally". - CHECK(GetPrimitiveType(to.Get(), &type)); - // First perform a primitive conversion to the unboxed equivalent of the target, - // if necessary. This should be for the rarer cases like (int->Long) etc. - if (UNLIKELY(from_type != type)) { - if (!ConvertPrimitiveValue(false, from_type, type, from_value, to_value)) { - return false; - } - } else { - *to_value = from_value; - } - - // Then perform the actual boxing, and then set the reference. Note that - // BoxPrimitive can return null if an OOM occurs. - ObjPtr<mirror::Object> boxed = BoxPrimitive(type, from_value); - if (boxed.Ptr() == nullptr) { - DCHECK(Thread::Current()->IsExceptionPending()); - return false; - } - - to_value->SetL(boxed.Ptr()); - return true; - } else { - // The target type is a primitive, we must unbox. - ObjPtr<mirror::Object> ref(from_value.GetL()); - - // Note that UnboxPrimitiveForResult already performs all of the type - // conversions that we want, based on |to|. - JValue unboxed_value; - return UnboxPrimitiveForResult(ref, to.Get(), to_value); - } +inline bool ConvertReturnValue(Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, + JValue* value) REQUIRES_SHARED(Locks::mutator_lock_) { + ObjPtr<mirror::Class> from_class(callee_type->GetRType()); + ObjPtr<mirror::Class> to_class(callsite_type->GetRType()); + if (to_class->GetPrimitiveType() == Primitive::kPrimVoid || from_class == to_class) { + return true; } - return true; + // |value| may contain a bare heap pointer which is generally + // unsafe. ConvertJValueCommon() saves |value|, |from_class|, and + // |to_class| to Handles where necessary to avoid issues if the heap + // changes. + if (ConvertJValueCommon(callsite_type, callee_type, from_class, to_class, value)) { + DCHECK(!Thread::Current()->IsExceptionPending()); + return true; + } else { + DCHECK(Thread::Current()->IsExceptionPending()); + value->SetJ(0); + return false; + } } template <typename G, typename S> bool PerformConversions(Thread* self, - Handle<mirror::ObjectArray<mirror::Class>> from_types, - Handle<mirror::ObjectArray<mirror::Class>> to_types, + Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, G* getter, S* setter, - int32_t num_conversions) { + int32_t num_conversions) REQUIRES_SHARED(Locks::mutator_lock_) { StackHandleScope<2> hs(self); - MutableHandle<mirror::Class> from(hs.NewHandle<mirror::Class>(nullptr)); - MutableHandle<mirror::Class> to(hs.NewHandle<mirror::Class>(nullptr)); + Handle<mirror::ObjectArray<mirror::Class>> from_types(hs.NewHandle(callsite_type->GetPTypes())); + Handle<mirror::ObjectArray<mirror::Class>> to_types(hs.NewHandle(callee_type->GetPTypes())); for (int32_t i = 0; i < num_conversions; ++i) { - from.Assign(from_types->GetWithoutChecks(i)); - to.Assign(to_types->GetWithoutChecks(i)); - - const Primitive::Type from_type = from->GetPrimitiveType(); - const Primitive::Type to_type = to->GetPrimitiveType(); - - if (from.Get() == to.Get()) { + ObjPtr<mirror::Class> from(from_types->GetWithoutChecks(i)); + ObjPtr<mirror::Class> to(to_types->GetWithoutChecks(i)); + const Primitive::Type from_type = from_types->GetWithoutChecks(i)->GetPrimitiveType(); + const Primitive::Type to_type = to_types->GetWithoutChecks(i)->GetPrimitiveType(); + if (from == to) { // Easy case - the types are identical. Nothing left to do except to pass // the arguments along verbatim. if (Primitive::Is64BitType(from_type)) { @@ -162,28 +105,28 @@ bool PerformConversions(Thread* self, setter->Set(getter->Get()); } } else { - JValue from_value; - JValue to_value; + JValue value; if (Primitive::Is64BitType(from_type)) { - from_value.SetJ(getter->GetLong()); + value.SetJ(getter->GetLong()); } else if (from_type == Primitive::kPrimNot) { - from_value.SetL(getter->GetReference()); + value.SetL(getter->GetReference()); } else { - from_value.SetI(getter->Get()); + value.SetI(getter->Get()); } - if (!ConvertJValue(from, to, from_value, &to_value)) { + // Caveat emptor - ObjPtr's not guaranteed valid after this call. + if (!ConvertArgumentValue(callsite_type, callee_type, i, &value)) { DCHECK(self->IsExceptionPending()); return false; } if (Primitive::Is64BitType(to_type)) { - setter->SetLong(to_value.GetJ()); + setter->SetLong(value.GetJ()); } else if (to_type == Primitive::kPrimNot) { - setter->SetReference(to_value.GetL()); + setter->SetReference(value.GetL()); } else { - setter->Set(to_value.GetI()); + setter->Set(value.GetI()); } } } @@ -199,10 +142,10 @@ bool ConvertAndCopyArgumentsFromCallerFrame(Thread* self, uint32_t first_src_reg, uint32_t first_dest_reg, const uint32_t (&arg)[Instruction::kMaxVarArgRegs], - ShadowFrame* callee_frame) { - StackHandleScope<4> hs(self); - Handle<mirror::ObjectArray<mirror::Class>> from_types(hs.NewHandle(callsite_type->GetPTypes())); - Handle<mirror::ObjectArray<mirror::Class>> to_types(hs.NewHandle(callee_type->GetPTypes())); + ShadowFrame* callee_frame) + REQUIRES_SHARED(Locks::mutator_lock_) { + ObjPtr<mirror::ObjectArray<mirror::Class>> from_types(callsite_type->GetPTypes()); + ObjPtr<mirror::ObjectArray<mirror::Class>> to_types(callee_type->GetPTypes()); const int32_t num_method_params = from_types->GetLength(); if (to_types->GetLength() != num_method_params) { @@ -214,8 +157,8 @@ bool ConvertAndCopyArgumentsFromCallerFrame(Thread* self, ShadowFrameSetter setter(callee_frame, first_dest_reg); return PerformConversions<ShadowFrameGetter<is_range>, ShadowFrameSetter>(self, - from_types, - to_types, + callsite_type, + callee_type, &getter, &setter, num_method_params); diff --git a/runtime/method_handles.cc b/runtime/method_handles.cc new file mode 100644 index 0000000000..491d13926f --- /dev/null +++ b/runtime/method_handles.cc @@ -0,0 +1,317 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "method_handles.h" + +#include "method_handles-inl.h" +#include "jvalue.h" +#include "jvalue-inl.h" +#include "reflection.h" +#include "reflection-inl.h" + +namespace art { + +namespace { + +static const char* kBoxedBooleanClass = "Ljava/lang/Boolean;"; +static const char* kBoxedByteClass = "Ljava/lang/Byte;"; +static const char* kBoxedCharacterClass = "Ljava/lang/Character;"; +static const char* kBoxedDoubleClass = "Ljava/lang/Double;"; +static const char* kBoxedFloatClass = "Ljava/lang/Float;"; +static const char* kBoxedIntegerClass = "Ljava/lang/Integer;"; +static const char* kBoxedLongClass = "Ljava/lang/Long;"; +static const char* kBoxedShortClass = "Ljava/lang/Short;"; + +// Assigns |type| to the primitive type associated with |klass|. Returns +// true iff. |klass| was a boxed type (Integer, Long etc.), false otherwise. +bool GetUnboxedPrimitiveType(ObjPtr<mirror::Class> klass, Primitive::Type* type) + REQUIRES_SHARED(Locks::mutator_lock_) { + ScopedAssertNoThreadSuspension ants(__FUNCTION__); + if (klass->DescriptorEquals(kBoxedBooleanClass)) { + (*type) = Primitive::kPrimBoolean; + return true; + } else if (klass->DescriptorEquals(kBoxedByteClass)) { + (*type) = Primitive::kPrimByte; + return true; + } else if (klass->DescriptorEquals(kBoxedCharacterClass)) { + (*type) = Primitive::kPrimChar; + return true; + } else if (klass->DescriptorEquals(kBoxedFloatClass)) { + (*type) = Primitive::kPrimFloat; + return true; + } else if (klass->DescriptorEquals(kBoxedDoubleClass)) { + (*type) = Primitive::kPrimDouble; + return true; + } else if (klass->DescriptorEquals(kBoxedIntegerClass)) { + (*type) = Primitive::kPrimInt; + return true; + } else if (klass->DescriptorEquals(kBoxedLongClass)) { + (*type) = Primitive::kPrimLong; + return true; + } else if (klass->DescriptorEquals(kBoxedShortClass)) { + (*type) = Primitive::kPrimShort; + return true; + } else { + return false; + } +} + +// Returns the class corresponding to the boxed type for the primitive |type|. +ObjPtr<mirror::Class> GetBoxedPrimitiveClass(Primitive::Type type) + REQUIRES_SHARED(Locks::mutator_lock_) { + ScopedAssertNoThreadSuspension ants(__FUNCTION__); + ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); + switch (type) { + case Primitive::kPrimBoolean: + return class_linker->FindSystemClass(Thread::Current(), kBoxedBooleanClass); + case Primitive::kPrimByte: + return class_linker->FindSystemClass(Thread::Current(), kBoxedByteClass); + case Primitive::kPrimChar: + return class_linker->FindSystemClass(Thread::Current(), kBoxedCharacterClass); + case Primitive::kPrimShort: + return class_linker->FindSystemClass(Thread::Current(), kBoxedShortClass); + case Primitive::kPrimInt: + return class_linker->FindSystemClass(Thread::Current(), kBoxedIntegerClass); + case Primitive::kPrimLong: + return class_linker->FindSystemClass(Thread::Current(), kBoxedLongClass); + case Primitive::kPrimFloat: + return class_linker->FindSystemClass(Thread::Current(), kBoxedFloatClass); + case Primitive::kPrimDouble: + return class_linker->FindSystemClass(Thread::Current(), kBoxedDoubleClass); + case Primitive::kPrimNot: + case Primitive::kPrimVoid: + LOG(FATAL) << "Unreachable"; + return nullptr; + } +} + +// Returns true if |klass| is a boxed primitive type or a sub-class of a boxed primitive type. +bool IsSubClassOfBoxedPrimitive(const Handle<mirror::Class>& klass) + REQUIRES_SHARED(Locks::mutator_lock_) { + StackHandleScope<1> hs(Thread::Current()); + MutableHandle<mirror::Class> h_klass(hs.NewHandle(klass.Get())); + do { + Primitive::Type type; + if (GetUnboxedPrimitiveType(h_klass.Get(), &type)) { + return true; + } + h_klass.Assign(h_klass->GetSuperClass()); + } while (h_klass.Get() != nullptr); + return false; +} + +// Unboxed the value |o| to |unboxed_value| of type |dst_class|. +// |unboxed_value| must be zero on entry to avoid dangling pointers. +// Returns true on success, false if an exception is raised. +bool UnboxPrimitiveForMethodHandles(ObjPtr<mirror::Object> o, + ObjPtr<mirror::Class> dst_class, + JValue* unboxed_value) + REQUIRES_SHARED(Locks::mutator_lock_) { + // Check unboxed_value does not contain a dangling pointer. + DCHECK_EQ(unboxed_value->GetJ(), 0); + DCHECK(dst_class->IsPrimitive()); + + // This is derived from UnboxPrimitive() in reflection.cc, but with + // exceptions appropriate to method handles. + if (UNLIKELY(dst_class->GetPrimitiveType() == Primitive::kPrimVoid)) { + ThrowClassCastException(o->GetClass(), dst_class); + return false; + } + if (UNLIKELY(o == nullptr)) { + ThrowNullPointerException( + StringPrintf("Expected to unbox a '%s' primitive type but was returned null", + dst_class->PrettyDescriptor().c_str()).c_str()); + return false; + } + + JValue boxed_value; + ObjPtr<mirror::Class> klass = o->GetClass(); + ObjPtr<mirror::Class> src_class = nullptr; + ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); + ArtField* primitive_field = &klass->GetIFieldsPtr()->At(0); + if (klass->DescriptorEquals(kBoxedBooleanClass)) { + src_class = class_linker->FindPrimitiveClass('Z'); + boxed_value.SetZ(primitive_field->GetBoolean(o)); + } else if (klass->DescriptorEquals(kBoxedByteClass)) { + src_class = class_linker->FindPrimitiveClass('B'); + boxed_value.SetB(primitive_field->GetByte(o)); + } else if (klass->DescriptorEquals(kBoxedCharacterClass)) { + src_class = class_linker->FindPrimitiveClass('C'); + boxed_value.SetC(primitive_field->GetChar(o)); + } else if (klass->DescriptorEquals(kBoxedFloatClass)) { + src_class = class_linker->FindPrimitiveClass('F'); + boxed_value.SetF(primitive_field->GetFloat(o)); + } else if (klass->DescriptorEquals(kBoxedDoubleClass)) { + src_class = class_linker->FindPrimitiveClass('D'); + boxed_value.SetD(primitive_field->GetDouble(o)); + } else if (klass->DescriptorEquals(kBoxedIntegerClass)) { + src_class = class_linker->FindPrimitiveClass('I'); + boxed_value.SetI(primitive_field->GetInt(o)); + } else if (klass->DescriptorEquals(kBoxedLongClass)) { + src_class = class_linker->FindPrimitiveClass('J'); + boxed_value.SetJ(primitive_field->GetLong(o)); + } else if (klass->DescriptorEquals(kBoxedShortClass)) { + src_class = class_linker->FindPrimitiveClass('S'); + boxed_value.SetS(primitive_field->GetShort(o)); + } else { + std::string temp; + ThrowIllegalArgumentException( + StringPrintf("result has type %s, got %s", + dst_class->PrettyDescriptor().c_str(), + PrettyDescriptor(o->GetClass()->GetDescriptor(&temp)).c_str()).c_str()); + return false; + } + + if (!ConvertPrimitiveValueNoThrow(src_class->GetPrimitiveType(), + dst_class->GetPrimitiveType(), + boxed_value, + unboxed_value)) { + ThrowClassCastException(src_class, dst_class); + return false; + } + return true; +} + +inline bool IsReferenceType(Primitive::Type type) { + return type == Primitive::kPrimNot; +} + +inline bool IsPrimitiveType(Primitive::Type type) { + return !IsReferenceType(type); +} + +} // namespace + +bool ConvertJValueCommon( + Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, + ObjPtr<mirror::Class> from, + ObjPtr<mirror::Class> to, + JValue* value) { + // The reader maybe concerned about the safety of the heap object + // that may be in |value|. There is only one case where allocation + // is obviously needed and that's for boxing. However, in the case + // of boxing |value| contains a non-reference type. + + const Primitive::Type from_type = from->GetPrimitiveType(); + const Primitive::Type to_type = to->GetPrimitiveType(); + + // This method must be called only when the types don't match. + DCHECK(from != to); + + if (IsPrimitiveType(from_type) && IsPrimitiveType(to_type)) { + // The source and target types are both primitives. + if (UNLIKELY(!ConvertPrimitiveValueNoThrow(from_type, to_type, *value, value))) { + ThrowWrongMethodTypeException(callee_type.Get(), callsite_type.Get()); + value->SetJ(0); + return false; + } + return true; + } else if (IsReferenceType(from_type) && IsReferenceType(to_type)) { + // They're both reference types. If "from" is null, we can pass it + // through unchanged. If not, we must generate a cast exception if + // |to| is not assignable from the dynamic type of |ref|. + // + // Playing it safe with StackHandleScope here, not expecting any allocation + // in mirror::Class::IsAssignable(). + StackHandleScope<2> hs(Thread::Current()); + Handle<mirror::Class> h_to(hs.NewHandle(to)); + Handle<mirror::Object> h_obj(hs.NewHandle(value->GetL())); + + // |value| will now be the result value, invalidate its existing value + // as |h_obj| now owns it. + value->SetJ(0); + + if (h_obj.Get() != nullptr && !to->IsAssignableFrom(h_obj->GetClass())) { + ThrowClassCastException(h_to.Get(), h_obj->GetClass()); + return false; + } + value->SetL(h_obj.Get()); + return true; + } else if (IsReferenceType(to_type)) { + DCHECK(IsPrimitiveType(from_type)); + // Playing it safe with StackHandleScope here with regards to + // GetUnboxedPrimitiveType() and GetBoxedPrimitiveClass(). + StackHandleScope<1> hs(Thread::Current()); + Handle<mirror::Class> h_to(hs.NewHandle(to)); + // The source type is a primitive and the target type is a reference, so we must box. + // The target type maybe a super class of the boxed source type, for example, + // if the source type is int, it's boxed type is java.lang.Integer, and the target + // type could be java.lang.Number. + Primitive::Type type; + if (!GetUnboxedPrimitiveType(to, &type)) { + ObjPtr<mirror::Class> boxed_from_class = GetBoxedPrimitiveClass(from_type); + if (boxed_from_class->IsSubClass(h_to.Get())) { + type = from_type; + } else { + value->SetJ(0); + ThrowWrongMethodTypeException(callee_type.Get(), callsite_type.Get()); + return false; + } + } + + if (UNLIKELY(from_type != type)) { + value->SetJ(0); + ThrowWrongMethodTypeException(callee_type.Get(), callsite_type.Get()); + return false; + } + + if (!ConvertPrimitiveValueNoThrow(from_type, type, *value, value)) { + value->SetJ(0); + ThrowWrongMethodTypeException(callee_type.Get(), callsite_type.Get()); + return false; + } + + // Then perform the actual boxing, and then set the reference. + ObjPtr<mirror::Object> boxed = BoxPrimitive(type, *value); + value->SetL(boxed.Ptr()); + return true; + } else { + // The source type is a reference and the target type is a primitive, so we must unbox. + DCHECK(IsReferenceType(from_type)); + DCHECK(IsPrimitiveType(to_type)); + + // Use StackHandleScope to protect |from|, |to|, and the reference + // in |value| from heap re-arrangements that could be triggered + // ahead of unboxing step. + StackHandleScope<3> hs(Thread::Current()); + Handle<mirror::Class> h_to(hs.NewHandle(to)); + Handle<mirror::Class> h_from(hs.NewHandle(from)); + Handle<mirror::Object> h_obj(hs.NewHandle(value->GetL())); + + // |value| will now be the result value, invalidate its existing value + // as |h_obj| now owns it. + value->SetJ(0); + + // Check source type is a boxed primitive or has a boxed primitive super-class. + ObjPtr<mirror::Class> boxed_to_class = GetBoxedPrimitiveClass(to_type); + if (!IsSubClassOfBoxedPrimitive(h_from) && !boxed_to_class->IsSubClass(h_from.Get())) { + ThrowWrongMethodTypeException(callee_type.Get(), callsite_type.Get()); + return false; + } + + if (h_obj.Get() == nullptr) { + ThrowNullPointerException( + StringPrintf("Expected to unbox a '%s' but instance was null", + h_from->PrettyDescriptor().c_str()).c_str()); + return false; + } + + return UnboxPrimitiveForMethodHandles(h_obj.Get(), h_to.Get(), value); + } +} + +} // namespace art diff --git a/runtime/method_handles.h b/runtime/method_handles.h index 06509bf13a..0cc69f2046 100644 --- a/runtime/method_handles.h +++ b/runtime/method_handles.h @@ -20,7 +20,10 @@ #include <ostream> #include "dex_instruction.h" +#include "handle.h" #include "jvalue.h" +#include "mirror/class.h" +#include "mirror/method_type.h" namespace art { @@ -56,12 +59,36 @@ inline bool IsInvoke(const MethodHandleKind handle_kind) { return handle_kind <= kLastInvokeKind; } -// Performs a single argument conversion from type |from| to a distinct -// type |to|. Returns true on success, false otherwise. -ALWAYS_INLINE bool ConvertJValue(Handle<mirror::Class> from, - Handle<mirror::Class> to, - const JValue& from_value, - JValue* to_value) REQUIRES_SHARED(Locks::mutator_lock_); +// Performs a conversion from type |from| to a distinct type |to| as +// part of conversion of |caller_type| to |callee_type|. The value to +// be converted is in |value|. Returns true on success and updates +// |value| with the converted value, false otherwise. +bool ConvertJValueCommon(Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, + ObjPtr<mirror::Class> from, + ObjPtr<mirror::Class> to, + JValue* value) + REQUIRES_SHARED(Locks::mutator_lock_); + +// Converts the value of the argument at position |index| from type +// expected by |callee_type| to type used by |callsite_type|. |value| +// represents the value to be converted. Returns true on success and +// updates |value|, false otherwise. +ALWAYS_INLINE bool ConvertArgumentValue(Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, + int index, + JValue* value) + REQUIRES_SHARED(Locks::mutator_lock_); + +// Converts the return value from return type yielded by +// |callee_type| to the return type yielded by +// |callsite_type|. |value| represents the value to be +// converted. Returns true on success and updates |value|, false +// otherwise. +ALWAYS_INLINE bool ConvertReturnValue(Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, + JValue* value) + REQUIRES_SHARED(Locks::mutator_lock_); // Perform argument conversions between |callsite_type| (the type of the // incoming arguments) and |callee_type| (the type of the method being @@ -109,8 +136,8 @@ ALWAYS_INLINE bool ConvertJValue(Handle<mirror::Class> from, // overridden by concrete classes. template <typename G, typename S> bool PerformConversions(Thread* self, - Handle<mirror::ObjectArray<mirror::Class>> from_types, - Handle<mirror::ObjectArray<mirror::Class>> to_types, + Handle<mirror::MethodType> callsite_type, + Handle<mirror::MethodType> callee_type, G* getter, S* setter, int32_t num_conversions) REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc index 03d648764f..db46027bc8 100644 --- a/runtime/mirror/class.cc +++ b/runtime/mirror/class.cc @@ -64,19 +64,45 @@ ClassExt* Class::GetExtData() { return GetFieldObject<ClassExt>(OFFSET_OF_OBJECT_MEMBER(Class, ext_data_)); } -void Class::SetExtData(ObjPtr<ClassExt> ext) { - CHECK(ext != nullptr) << PrettyClass(); - // TODO It might be wise to just create an internal (global?) mutex that we synchronize on instead - // to prevent any possibility of deadlocks with java code. Alternatively we might want to come up - // with some other abstraction. - DCHECK_EQ(GetLockOwnerThreadId(), Thread::Current()->GetThreadId()) - << "The " << PrettyClass() << " object should be locked when writing to the extData field."; - DCHECK(GetExtData() == nullptr) - << "The extData for " << PrettyClass() << " has already been set!"; - if (Runtime::Current()->IsActiveTransaction()) { - SetFieldObject<true>(OFFSET_OF_OBJECT_MEMBER(Class, ext_data_), ext); +ClassExt* Class::EnsureExtDataPresent(Thread* self) { + ObjPtr<ClassExt> existing(GetExtData()); + if (!existing.IsNull()) { + return existing.Ptr(); + } + StackHandleScope<3> hs(self); + // Handlerize 'this' since we are allocating here. + Handle<Class> h_this(hs.NewHandle(this)); + // Clear exception so we can allocate. + Handle<Throwable> throwable(hs.NewHandle(self->GetException())); + self->ClearException(); + // Allocate the ClassExt + Handle<ClassExt> new_ext(hs.NewHandle(ClassExt::Alloc(self))); + if (new_ext.Get() == nullptr) { + // OOM allocating the classExt. + // TODO Should we restore the suppressed exception? + self->AssertPendingOOMException(); + return nullptr; } else { - SetFieldObject<false>(OFFSET_OF_OBJECT_MEMBER(Class, ext_data_), ext); + MemberOffset ext_offset(OFFSET_OF_OBJECT_MEMBER(Class, ext_data_)); + bool set; + // Set the ext_data_ field using CAS semantics. + if (Runtime::Current()->IsActiveTransaction()) { + set = h_this->CasFieldStrongSequentiallyConsistentObject<true>(ext_offset, + ObjPtr<ClassExt>(nullptr), + new_ext.Get()); + } else { + set = h_this->CasFieldStrongSequentiallyConsistentObject<false>(ext_offset, + ObjPtr<ClassExt>(nullptr), + new_ext.Get()); + } + ObjPtr<ClassExt> ret(set ? new_ext.Get() : h_this->GetExtData()); + DCHECK(!set || h_this->GetExtData() == new_ext.Get()); + CHECK(!ret.IsNull()); + // Restore the exception if there was one. + if (throwable.Get() != nullptr) { + self->SetException(throwable.Get()); + } + return ret.Ptr(); } } @@ -108,34 +134,16 @@ void Class::SetStatus(Handle<Class> h_this, Status new_status, Thread* self) { } } - { - // Ensure we lock around 'this' when we set the ClassExt. - ObjectLock<mirror::Class> lock(self, h_this); - StackHandleScope<2> hs(self); - // Remember the current exception. - Handle<Throwable> exception(hs.NewHandle(self->GetException())); - CHECK(exception.Get() != nullptr); - MutableHandle<ClassExt> ext(hs.NewHandle(h_this->GetExtData())); - if (ext.Get() == nullptr) { - // Cannot have exception while allocating. - self->ClearException(); - ext.Assign(ClassExt::Alloc(self)); - DCHECK(ext.Get() == nullptr || ext->GetVerifyError() == nullptr); - if (ext.Get() != nullptr) { - self->AssertNoPendingException(); - h_this->SetExtData(ext.Get()); - self->SetException(exception.Get()); - } else { - // TODO Should we restore the old exception anyway? - self->AssertPendingOOMException(); - } - } - if (ext.Get() != nullptr) { - ext->SetVerifyError(self->GetException()); - } + ObjPtr<ClassExt> ext(h_this->EnsureExtDataPresent(self)); + if (!ext.IsNull()) { + self->AssertPendingException(); + ext->SetVerifyError(self->GetException()); + } else { + self->AssertPendingOOMException(); } self->AssertPendingException(); } + static_assert(sizeof(Status) == sizeof(uint32_t), "Size of status not equal to uint32"); if (Runtime::Current()->IsActiveTransaction()) { h_this->SetField32Volatile<true>(StatusOffset(), new_status); diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h index 23c70ff6b3..711914dc61 100644 --- a/runtime/mirror/class.h +++ b/runtime/mirror/class.h @@ -1133,6 +1133,12 @@ class MANAGED Class FINAL : public Object { ClassExt* GetExtData() REQUIRES_SHARED(Locks::mutator_lock_); + // Returns the ExtData for this class, allocating one if necessary. This should be the only way + // to force ext_data_ to be set. No functions are available for changing an already set ext_data_ + // since doing so is not allowed. + ClassExt* EnsureExtDataPresent(Thread* self) + REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Roles::uninterruptible_); + uint16_t GetDexClassDefIndex() REQUIRES_SHARED(Locks::mutator_lock_) { return GetField32(OFFSET_OF_OBJECT_MEMBER(Class, dex_class_def_idx_)); } @@ -1320,9 +1326,6 @@ class MANAGED Class FINAL : public Object { ALWAYS_INLINE void SetMethodsPtrInternal(LengthPrefixedArray<ArtMethod>* new_methods) REQUIRES_SHARED(Locks::mutator_lock_); - // Set the extData field. This should be done while the 'this' is locked to prevent races. - void SetExtData(ObjPtr<ClassExt> ext) REQUIRES_SHARED(Locks::mutator_lock_); - template <bool throw_on_failure, bool use_referrers_cache> bool ResolvedFieldAccessTest(ObjPtr<Class> access_to, ArtField* field, diff --git a/runtime/mirror/emulated_stack_frame.cc b/runtime/mirror/emulated_stack_frame.cc index 861d416489..d607040486 100644 --- a/runtime/mirror/emulated_stack_frame.cc +++ b/runtime/mirror/emulated_stack_frame.cc @@ -188,7 +188,7 @@ mirror::EmulatedStackFrame* EmulatedStackFrame::CreateFromShadowFrameAndArgs( ShadowFrameGetter<is_range> getter(first_src_reg, arg, caller_frame); EmulatedStackFrameAccessor setter(references, stack_frame, stack_frame->GetLength()); if (!PerformConversions<ShadowFrameGetter<is_range>, EmulatedStackFrameAccessor>( - self, from_types, to_types, &getter, &setter, num_method_params)) { + self, caller_type, callee_type, &getter, &setter, num_method_params)) { return nullptr; } @@ -206,9 +206,8 @@ bool EmulatedStackFrame::WriteToShadowFrame(Thread* self, Handle<mirror::MethodType> callee_type, const uint32_t first_dest_reg, ShadowFrame* callee_frame) { - StackHandleScope<4> hs(self); - Handle<mirror::ObjectArray<mirror::Class>> from_types(hs.NewHandle(GetType()->GetPTypes())); - Handle<mirror::ObjectArray<mirror::Class>> to_types(hs.NewHandle(callee_type->GetPTypes())); + ObjPtr<mirror::ObjectArray<mirror::Class>> from_types(GetType()->GetPTypes()); + ObjPtr<mirror::ObjectArray<mirror::Class>> to_types(callee_type->GetPTypes()); const int32_t num_method_params = from_types->GetLength(); if (to_types->GetLength() != num_method_params) { @@ -216,6 +215,8 @@ bool EmulatedStackFrame::WriteToShadowFrame(Thread* self, return false; } + StackHandleScope<3> hs(self); + Handle<mirror::MethodType> frame_callsite_type(hs.NewHandle(GetType())); Handle<mirror::ObjectArray<mirror::Object>> references(hs.NewHandle(GetReferences())); Handle<ByteArray> stack_frame(hs.NewHandle(GetStackFrame())); @@ -223,7 +224,7 @@ bool EmulatedStackFrame::WriteToShadowFrame(Thread* self, ShadowFrameSetter setter(callee_frame, first_dest_reg); return PerformConversions<EmulatedStackFrameAccessor, ShadowFrameSetter>( - self, from_types, to_types, &getter, &setter, num_method_params); + self, frame_callsite_type, callee_type, &getter, &setter, num_method_params); } void EmulatedStackFrame::GetReturnValue(Thread* self, JValue* value) { diff --git a/runtime/mirror/method_handle_impl.h b/runtime/mirror/method_handle_impl.h index 7bf9c5b24c..9054216065 100644 --- a/runtime/mirror/method_handle_impl.h +++ b/runtime/mirror/method_handle_impl.h @@ -36,6 +36,10 @@ class MANAGED MethodHandle : public Object { return GetFieldObject<mirror::MethodType>(OFFSET_OF_OBJECT_MEMBER(MethodHandle, method_type_)); } + mirror::MethodType* GetNominalType() REQUIRES_SHARED(Locks::mutator_lock_) { + return GetFieldObject<mirror::MethodType>(OFFSET_OF_OBJECT_MEMBER(MethodHandle, nominal_type_)); + } + ArtField* GetTargetField() REQUIRES_SHARED(Locks::mutator_lock_) { return reinterpret_cast<ArtField*>( GetField64(OFFSET_OF_OBJECT_MEMBER(MethodHandle, art_field_or_method_))); @@ -54,14 +58,14 @@ class MANAGED MethodHandle : public Object { } private: - HeapReference<mirror::Object> as_type_cache_; + HeapReference<mirror::MethodType> nominal_type_; HeapReference<mirror::MethodType> method_type_; uint64_t art_field_or_method_; uint32_t handle_kind_; private: - static MemberOffset AsTypeCacheOffset() { - return MemberOffset(OFFSETOF_MEMBER(MethodHandle, as_type_cache_)); + static MemberOffset NominalTypeOffset() { + return MemberOffset(OFFSETOF_MEMBER(MethodHandle, nominal_type_)); } static MemberOffset MethodTypeOffset() { return MemberOffset(OFFSETOF_MEMBER(MethodHandle, method_type_)); diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h index 3bf9d94410..6d29ed379d 100644 --- a/runtime/mirror/object-inl.h +++ b/runtime/mirror/object-inl.h @@ -135,25 +135,82 @@ inline void Object::Wait(Thread* self, int64_t ms, int32_t ns) { Monitor::Wait(self, this, ms, ns, true, kTimedWaiting); } -inline Object* Object::GetReadBarrierPointer() { +inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) { +#ifdef USE_BAKER_READ_BARRIER + CHECK(kUseBakerReadBarrier); +#if defined(__arm__) + uintptr_t obj = reinterpret_cast<uintptr_t>(this); + uintptr_t result; + DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U); + // Use inline assembly to prevent the compiler from optimizing away the false dependency. + __asm__ __volatile__( + "ldr %[result], [%[obj], #4]\n\t" + // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be + // null, without them being able to assume that fact. + "eor %[fad], %[result], %[result]\n\t" + : [result] "+r" (result), [fad] "=r" (*fake_address_dependency) + : [obj] "r" (obj)); + DCHECK_EQ(*fake_address_dependency, 0U); + LockWord lw(static_cast<uint32_t>(result)); + uint32_t rb_state = lw.ReadBarrierState(); + return rb_state; +#elif defined(__aarch64__) + uintptr_t obj = reinterpret_cast<uintptr_t>(this); + uintptr_t result; + DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U); + // Use inline assembly to prevent the compiler from optimizing away the false dependency. + __asm__ __volatile__( + "ldr %w[result], [%[obj], #4]\n\t" + // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be + // null, without them being able to assume that fact. + "eor %[fad], %[result], %[result]\n\t" + : [result] "+r" (result), [fad] "=r" (*fake_address_dependency) + : [obj] "r" (obj)); + DCHECK_EQ(*fake_address_dependency, 0U); + LockWord lw(static_cast<uint32_t>(result)); + uint32_t rb_state = lw.ReadBarrierState(); + return rb_state; +#elif defined(__i386__) || defined(__x86_64__) + LockWord lw = GetLockWord(false); + // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler + // reordering. + *fake_address_dependency = 0; + std::atomic_signal_fence(std::memory_order_acquire); + uint32_t rb_state = lw.ReadBarrierState(); + return rb_state; +#else + // mips/mips64 + LOG(FATAL) << "Unreachable"; + UNREACHABLE(); + UNUSED(fake_address_dependency); +#endif +#else // !USE_BAKER_READ_BARRIER + LOG(FATAL) << "Unreachable"; + UNREACHABLE(); + UNUSED(fake_address_dependency); +#endif +} + +inline uint32_t Object::GetReadBarrierState() { #ifdef USE_BAKER_READ_BARRIER DCHECK(kUseBakerReadBarrier); - return reinterpret_cast<Object*>(GetLockWord(false).ReadBarrierState()); -#elif USE_BROOKS_READ_BARRIER - DCHECK(kUseBrooksReadBarrier); - return GetFieldObject<Object, kVerifyNone, kWithoutReadBarrier>( - OFFSET_OF_OBJECT_MEMBER(Object, x_rb_ptr_)); + LockWord lw(GetField<uint32_t, /*kIsVolatile*/false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); + uint32_t rb_state = lw.ReadBarrierState(); + DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; + return rb_state; #else LOG(FATAL) << "Unreachable"; UNREACHABLE(); #endif } -inline Object* Object::GetReadBarrierPointerAcquire() { +inline uint32_t Object::GetReadBarrierStateAcquire() { #ifdef USE_BAKER_READ_BARRIER DCHECK(kUseBakerReadBarrier); LockWord lw(GetFieldAcquire<uint32_t>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_))); - return reinterpret_cast<Object*>(lw.ReadBarrierState()); + uint32_t rb_state = lw.ReadBarrierState(); + DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; + return rb_state; #else LOG(FATAL) << "Unreachable"; UNREACHABLE(); @@ -169,48 +226,38 @@ inline uint32_t Object::GetMarkBit() { #endif } -inline void Object::SetReadBarrierPointer(Object* rb_ptr) { +inline void Object::SetReadBarrierState(uint32_t rb_state) { #ifdef USE_BAKER_READ_BARRIER DCHECK(kUseBakerReadBarrier); - DCHECK_EQ(reinterpret_cast<uint64_t>(rb_ptr) >> 32, 0U); - DCHECK_NE(rb_ptr, ReadBarrier::BlackPtr()) << "Setting to black is not supported"; + DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; LockWord lw = GetLockWord(false); - lw.SetReadBarrierState(static_cast<uint32_t>(reinterpret_cast<uintptr_t>(rb_ptr))); + lw.SetReadBarrierState(rb_state); SetLockWord(lw, false); -#elif USE_BROOKS_READ_BARRIER - DCHECK(kUseBrooksReadBarrier); - // We don't mark the card as this occurs as part of object allocation. Not all objects have - // backing cards, such as large objects. - SetFieldObjectWithoutWriteBarrier<false, false, kVerifyNone>( - OFFSET_OF_OBJECT_MEMBER(Object, x_rb_ptr_), rb_ptr); #else LOG(FATAL) << "Unreachable"; UNREACHABLE(); - UNUSED(rb_ptr); + UNUSED(rb_state); #endif } template<bool kCasRelease> -inline bool Object::AtomicSetReadBarrierPointer(Object* expected_rb_ptr, Object* rb_ptr) { +inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) { #ifdef USE_BAKER_READ_BARRIER DCHECK(kUseBakerReadBarrier); - DCHECK_EQ(reinterpret_cast<uint64_t>(expected_rb_ptr) >> 32, 0U); - DCHECK_EQ(reinterpret_cast<uint64_t>(rb_ptr) >> 32, 0U); - DCHECK_NE(expected_rb_ptr, ReadBarrier::BlackPtr()) << "Setting to black is not supported"; - DCHECK_NE(rb_ptr, ReadBarrier::BlackPtr()) << "Setting to black is not supported"; + DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state; + DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state; LockWord expected_lw; LockWord new_lw; do { LockWord lw = GetLockWord(false); - if (UNLIKELY(reinterpret_cast<Object*>(lw.ReadBarrierState()) != expected_rb_ptr)) { + if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) { // Lost the race. return false; } expected_lw = lw; - expected_lw.SetReadBarrierState( - static_cast<uint32_t>(reinterpret_cast<uintptr_t>(expected_rb_ptr))); + expected_lw.SetReadBarrierState(expected_rb_state); new_lw = lw; - new_lw.SetReadBarrierState(static_cast<uint32_t>(reinterpret_cast<uintptr_t>(rb_ptr))); + new_lw.SetReadBarrierState(rb_state); // ConcurrentCopying::ProcessMarkStackRef uses this with kCasRelease == true. // If kCasRelease == true, use a CAS release so that when GC updates all the fields of // an object and then changes the object from gray to black, the field updates (stores) will be @@ -219,23 +266,8 @@ inline bool Object::AtomicSetReadBarrierPointer(Object* expected_rb_ptr, Object* CasLockWordWeakRelease(expected_lw, new_lw) : CasLockWordWeakRelaxed(expected_lw, new_lw))); return true; -#elif USE_BROOKS_READ_BARRIER - DCHECK(kUseBrooksReadBarrier); - MemberOffset offset = OFFSET_OF_OBJECT_MEMBER(Object, x_rb_ptr_); - uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + offset.SizeValue(); - Atomic<uint32_t>* atomic_rb_ptr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - HeapReference<Object> expected_ref(HeapReference<Object>::FromMirrorPtr(expected_rb_ptr)); - HeapReference<Object> new_ref(HeapReference<Object>::FromMirrorPtr(rb_ptr)); - do { - if (UNLIKELY(atomic_rb_ptr->LoadRelaxed() != expected_ref.reference_)) { - // Lost the race. - return false; - } - } while (!atomic_rb_ptr->CompareExchangeWeakSequentiallyConsistent(expected_ref.reference_, - new_ref.reference_)); - return true; #else - UNUSED(expected_rb_ptr, rb_ptr); + UNUSED(expected_rb_state, rb_state); LOG(FATAL) << "Unreachable"; UNREACHABLE(); #endif @@ -259,19 +291,12 @@ inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_b } -inline void Object::AssertReadBarrierPointer() const { - if (kUseBakerReadBarrier) { - Object* obj = const_cast<Object*>(this); - DCHECK(obj->GetReadBarrierPointer() == nullptr) - << "Bad Baker pointer: obj=" << reinterpret_cast<void*>(obj) - << " ptr=" << reinterpret_cast<void*>(obj->GetReadBarrierPointer()); - } else { - CHECK(kUseBrooksReadBarrier); - Object* obj = const_cast<Object*>(this); - DCHECK_EQ(obj, obj->GetReadBarrierPointer()) - << "Bad Brooks pointer: obj=" << reinterpret_cast<void*>(obj) - << " ptr=" << reinterpret_cast<void*>(obj->GetReadBarrierPointer()); - } +inline void Object::AssertReadBarrierState() const { + CHECK(kUseBakerReadBarrier); + Object* obj = const_cast<Object*>(this); + DCHECK(obj->GetReadBarrierState() == ReadBarrier::WhiteState()) + << "Bad Baker pointer: obj=" << reinterpret_cast<void*>(obj) + << " rb_state" << reinterpret_cast<void*>(obj->GetReadBarrierState()); } template<VerifyObjectFlags kVerifyFlags> diff --git a/runtime/mirror/object.cc b/runtime/mirror/object.cc index 8cfb60e60e..f5b9ab36de 100644 --- a/runtime/mirror/object.cc +++ b/runtime/mirror/object.cc @@ -235,8 +235,6 @@ void Object::CheckFieldAssignmentImpl(MemberOffset field_offset, ObjPtr<Object> } for (ObjPtr<Class> cur = c; cur != nullptr; cur = cur->GetSuperClass()) { for (ArtField& field : cur->GetIFields()) { - StackHandleScope<1> hs(Thread::Current()); - Handle<Object> h_object(hs.NewHandle(new_value)); if (field.GetOffset().Int32Value() == field_offset.Int32Value()) { CHECK_NE(field.GetTypeAsPrimitiveType(), Primitive::kPrimNot); // TODO: resolve the field type for moving GC. diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h index 886637be5c..67b5ddbb32 100644 --- a/runtime/mirror/object.h +++ b/runtime/mirror/object.h @@ -94,19 +94,22 @@ class MANAGED LOCKABLE Object { template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> void SetClass(ObjPtr<Class> new_klass) REQUIRES_SHARED(Locks::mutator_lock_); - // TODO: Clean these up and change to return int32_t - Object* GetReadBarrierPointer() REQUIRES_SHARED(Locks::mutator_lock_); - - // Get the read barrier pointer with release semantics, only supported for baker. - Object* GetReadBarrierPointerAcquire() REQUIRES_SHARED(Locks::mutator_lock_); + // Get the read barrier state with a fake address dependency. + // '*fake_address_dependency' will be set to 0. + ALWAYS_INLINE uint32_t GetReadBarrierState(uintptr_t* fake_address_dependency) + REQUIRES_SHARED(Locks::mutator_lock_); + // This version does not offer any special mechanism to prevent load-load reordering. + ALWAYS_INLINE uint32_t GetReadBarrierState() REQUIRES_SHARED(Locks::mutator_lock_); + // Get the read barrier state with a load-acquire. + ALWAYS_INLINE uint32_t GetReadBarrierStateAcquire() REQUIRES_SHARED(Locks::mutator_lock_); #ifndef USE_BAKER_OR_BROOKS_READ_BARRIER NO_RETURN #endif - void SetReadBarrierPointer(Object* rb_ptr) REQUIRES_SHARED(Locks::mutator_lock_); + ALWAYS_INLINE void SetReadBarrierState(uint32_t rb_state) REQUIRES_SHARED(Locks::mutator_lock_); template<bool kCasRelease = false> - ALWAYS_INLINE bool AtomicSetReadBarrierPointer(Object* expected_rb_ptr, Object* rb_ptr) + ALWAYS_INLINE bool AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) REQUIRES_SHARED(Locks::mutator_lock_); ALWAYS_INLINE uint32_t GetMarkBit() REQUIRES_SHARED(Locks::mutator_lock_); @@ -114,7 +117,8 @@ class MANAGED LOCKABLE Object { ALWAYS_INLINE bool AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) REQUIRES_SHARED(Locks::mutator_lock_); - void AssertReadBarrierPointer() const REQUIRES_SHARED(Locks::mutator_lock_); + // Assert that the read barrier state is in the default (white) state. + ALWAYS_INLINE void AssertReadBarrierState() const REQUIRES_SHARED(Locks::mutator_lock_); // The verifier treats all interfaces as java.lang.Object and relies on runtime checks in // invoke-interface to detect incompatible interface types. diff --git a/runtime/monitor.cc b/runtime/monitor.cc index eb74fcf7f4..f5831670fa 100644 --- a/runtime/monitor.cc +++ b/runtime/monitor.cc @@ -771,7 +771,7 @@ bool Monitor::Deflate(Thread* self, mirror::Object* obj) { return false; } // Can't deflate if our lock count is too high. - if (monitor->lock_count_ > LockWord::kThinLockMaxCount) { + if (static_cast<uint32_t>(monitor->lock_count_) > LockWord::kThinLockMaxCount) { return false; } // Deflate to a thin lock. diff --git a/runtime/monitor_test.cc b/runtime/monitor_test.cc index 4ee46dcdff..4fbfe4781c 100644 --- a/runtime/monitor_test.cc +++ b/runtime/monitor_test.cc @@ -401,14 +401,11 @@ TEST_F(MonitorTest, TestTryLock) { Thread* const self = Thread::Current(); ThreadPool thread_pool("the pool", 2); ScopedObjectAccess soa(self); - StackHandleScope<3> hs(self); + StackHandleScope<1> hs(self); Handle<mirror::Object> obj1( hs.NewHandle<mirror::Object>(mirror::String::AllocFromModifiedUtf8(self, "hello, world!"))); - Handle<mirror::Object> obj2( - hs.NewHandle<mirror::Object>(mirror::String::AllocFromModifiedUtf8(self, "hello, world!"))); { ObjectLock<mirror::Object> lock1(self, obj1); - ObjectLock<mirror::Object> lock2(self, obj1); { ObjectTryLock<mirror::Object> trylock(self, obj1); EXPECT_TRUE(trylock.Acquired()); diff --git a/runtime/oat_file_manager.cc b/runtime/oat_file_manager.cc index fde0a2cc00..cf9efe0782 100644 --- a/runtime/oat_file_manager.cc +++ b/runtime/oat_file_manager.cc @@ -377,7 +377,7 @@ static void GetDexFilesFromDexElementsArray( } } -static bool AreSharedLibrariesOk(const std::string shared_libraries, +static bool AreSharedLibrariesOk(const std::string& shared_libraries, std::priority_queue<DexFileAndClassPair>& queue) { if (shared_libraries.empty()) { if (queue.empty()) { @@ -398,7 +398,7 @@ static bool AreSharedLibrariesOk(const std::string shared_libraries, while (!temp.empty() && index < shared_libraries_split.size() - 1) { DexFileAndClassPair pair(temp.top()); const DexFile* dex_file = pair.GetDexFile(); - std::string dex_filename(dex_file->GetLocation()); + const std::string& dex_filename = dex_file->GetLocation(); uint32_t dex_checksum = dex_file->GetLocationChecksum(); if (dex_filename != shared_libraries_split[index] || dex_checksum != std::stoul(shared_libraries_split[index + 1])) { diff --git a/runtime/obj_ptr.h b/runtime/obj_ptr.h index 93182323d6..d24c6fbd2c 100644 --- a/runtime/obj_ptr.h +++ b/runtime/obj_ptr.h @@ -20,6 +20,7 @@ #include <ostream> #include <type_traits> +#include "base/macros.h" #include "base/mutex.h" // For Locks::mutator_lock_. #include "globals.h" @@ -41,17 +42,26 @@ class ObjPtr { public: ALWAYS_INLINE ObjPtr() REQUIRES_SHARED(Locks::mutator_lock_) : reference_(0u) {} - ALWAYS_INLINE ObjPtr(std::nullptr_t) REQUIRES_SHARED(Locks::mutator_lock_) : reference_(0u) {} + // Note: The following constructors allow implicit conversion. This simplifies code that uses + // them, e.g., for parameter passing. However, in general, implicit-conversion constructors + // are discouraged and detected by cpplint and clang-tidy. So mark these constructors + // as NOLINT (without category, as the categories are different). + + ALWAYS_INLINE ObjPtr(std::nullptr_t) // NOLINT + REQUIRES_SHARED(Locks::mutator_lock_) + : reference_(0u) {} template <typename Type> - ALWAYS_INLINE ObjPtr(Type* ptr) REQUIRES_SHARED(Locks::mutator_lock_) + ALWAYS_INLINE ObjPtr(Type* ptr) // NOLINT + REQUIRES_SHARED(Locks::mutator_lock_) : reference_(Encode(static_cast<MirrorType*>(ptr))) { static_assert(std::is_base_of<MirrorType, Type>::value, "Input type must be a subtype of the ObjPtr type"); } template <typename Type> - ALWAYS_INLINE ObjPtr(const ObjPtr<Type, kPoison>& other) REQUIRES_SHARED(Locks::mutator_lock_) + ALWAYS_INLINE ObjPtr(const ObjPtr<Type, kPoison>& other) // NOLINT + REQUIRES_SHARED(Locks::mutator_lock_) : reference_(Encode(static_cast<MirrorType*>(other.Ptr()))) { static_assert(std::is_base_of<MirrorType, Type>::value, "Input type must be a subtype of the ObjPtr type"); @@ -154,6 +164,9 @@ class ObjPtr { uintptr_t reference_; }; +static_assert(std::is_trivially_copyable<ObjPtr<void>>::value, + "ObjPtr should be trivially copyable"); + // Hash function for stl data structures. class HashObjPtr { public: diff --git a/runtime/openjdkjvmti/object_tagging.cc b/runtime/openjdkjvmti/object_tagging.cc index b359f36316..b983e79658 100644 --- a/runtime/openjdkjvmti/object_tagging.cc +++ b/runtime/openjdkjvmti/object_tagging.cc @@ -47,6 +47,16 @@ namespace openjdkjvmti { +void ObjectTagTable::Lock() { + allow_disallow_lock_.ExclusiveLock(art::Thread::Current()); +} +void ObjectTagTable::Unlock() { + allow_disallow_lock_.ExclusiveUnlock(art::Thread::Current()); +} +void ObjectTagTable::AssertLocked() { + allow_disallow_lock_.AssertHeld(art::Thread::Current()); +} + void ObjectTagTable::UpdateTableWithReadBarrier() { update_since_last_sweep_ = true; @@ -80,6 +90,13 @@ bool ObjectTagTable::Remove(art::mirror::Object* obj, jlong* tag) { return RemoveLocked(self, obj, tag); } +bool ObjectTagTable::RemoveLocked(art::mirror::Object* obj, jlong* tag) { + art::Thread* self = art::Thread::Current(); + allow_disallow_lock_.AssertHeld(self); + Wait(self); + + return RemoveLocked(self, obj, tag); +} bool ObjectTagTable::RemoveLocked(art::Thread* self, art::mirror::Object* obj, jlong* tag) { auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj)); @@ -109,12 +126,29 @@ bool ObjectTagTable::RemoveLocked(art::Thread* self, art::mirror::Object* obj, j } bool ObjectTagTable::Set(art::mirror::Object* obj, jlong new_tag) { + if (new_tag == 0) { + jlong tmp; + return Remove(obj, &tmp); + } + art::Thread* self = art::Thread::Current(); art::MutexLock mu(self, allow_disallow_lock_); Wait(self); return SetLocked(self, obj, new_tag); } +bool ObjectTagTable::SetLocked(art::mirror::Object* obj, jlong new_tag) { + if (new_tag == 0) { + jlong tmp; + return RemoveLocked(obj, &tmp); + } + + art::Thread* self = art::Thread::Current(); + allow_disallow_lock_.AssertHeld(self); + Wait(self); + + return SetLocked(self, obj, new_tag); +} bool ObjectTagTable::SetLocked(art::Thread* self, art::mirror::Object* obj, jlong new_tag) { auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj)); diff --git a/runtime/openjdkjvmti/object_tagging.h b/runtime/openjdkjvmti/object_tagging.h index 071d139f18..997cedb093 100644 --- a/runtime/openjdkjvmti/object_tagging.h +++ b/runtime/openjdkjvmti/object_tagging.h @@ -46,10 +46,16 @@ class ObjectTagTable : public art::gc::SystemWeakHolder { bool Remove(art::mirror::Object* obj, jlong* tag) REQUIRES_SHARED(art::Locks::mutator_lock_) REQUIRES(!allow_disallow_lock_); + bool RemoveLocked(art::mirror::Object* obj, jlong* tag) + REQUIRES_SHARED(art::Locks::mutator_lock_) + REQUIRES(allow_disallow_lock_); bool Set(art::mirror::Object* obj, jlong tag) REQUIRES_SHARED(art::Locks::mutator_lock_) REQUIRES(!allow_disallow_lock_); + bool SetLocked(art::mirror::Object* obj, jlong tag) + REQUIRES_SHARED(art::Locks::mutator_lock_) + REQUIRES(allow_disallow_lock_); bool GetTag(art::mirror::Object* obj, jlong* result) REQUIRES_SHARED(art::Locks::mutator_lock_) @@ -60,6 +66,30 @@ class ObjectTagTable : public art::gc::SystemWeakHolder { return GetTagLocked(self, obj, result); } + bool GetTagLocked(art::mirror::Object* obj, jlong* result) + REQUIRES_SHARED(art::Locks::mutator_lock_) + REQUIRES(allow_disallow_lock_) { + art::Thread* self = art::Thread::Current(); + allow_disallow_lock_.AssertHeld(self); + Wait(self); + + return GetTagLocked(self, obj, result); + } + + jlong GetTagOrZero(art::mirror::Object* obj) + REQUIRES_SHARED(art::Locks::mutator_lock_) + REQUIRES(!allow_disallow_lock_) { + jlong tmp = 0; + GetTag(obj, &tmp); + return tmp; + } + jlong GetTagOrZeroLocked(art::mirror::Object* obj) + REQUIRES_SHARED(art::Locks::mutator_lock_) + REQUIRES(allow_disallow_lock_) { + jlong tmp = 0; + GetTagLocked(obj, &tmp); + return tmp; + } void Sweep(art::IsMarkedVisitor* visitor) REQUIRES_SHARED(art::Locks::mutator_lock_) @@ -74,6 +104,10 @@ class ObjectTagTable : public art::gc::SystemWeakHolder { REQUIRES_SHARED(art::Locks::mutator_lock_) REQUIRES(!allow_disallow_lock_); + void Lock() ACQUIRE(allow_disallow_lock_); + void Unlock() RELEASE(allow_disallow_lock_); + void AssertLocked() ASSERT_CAPABILITY(allow_disallow_lock_); + private: bool SetLocked(art::Thread* self, art::mirror::Object* obj, jlong tag) REQUIRES_SHARED(art::Locks::mutator_lock_) diff --git a/runtime/proxy_test.cc b/runtime/proxy_test.cc index 32a55822b7..fd7e56d726 100644 --- a/runtime/proxy_test.cc +++ b/runtime/proxy_test.cc @@ -199,8 +199,6 @@ TEST_F(ProxyTest, CheckArtMirrorFieldsOfProxyStaticFields) { ScopedObjectAccess soa(Thread::Current()); jobject jclass_loader = LoadDex("Interfaces"); StackHandleScope<7> hs(soa.Self()); - Handle<mirror::ClassLoader> class_loader( - hs.NewHandle(soa.Decode<mirror::ClassLoader>(jclass_loader))); Handle<mirror::Class> proxyClass0; Handle<mirror::Class> proxyClass1; diff --git a/runtime/read_barrier-inl.h b/runtime/read_barrier-inl.h index be95600528..37cf25755b 100644 --- a/runtime/read_barrier-inl.h +++ b/runtime/read_barrier-inl.h @@ -40,14 +40,16 @@ inline MirrorType* ReadBarrier::Barrier( } } if (kUseBakerReadBarrier) { - // The higher bits of the rb_ptr, rb_ptr_high_bits (must be zero) - // is used to create artificial data dependency from the is_gray - // load to the ref field (ptr) load to avoid needing a load-load - // barrier between the two. - uintptr_t rb_ptr_high_bits; - bool is_gray = HasGrayReadBarrierPointer(obj, &rb_ptr_high_bits); + // fake_address_dependency (must be zero) is used to create artificial data dependency from + // the is_gray load to the ref field (ptr) load to avoid needing a load-load barrier between + // the two. + uintptr_t fake_address_dependency; + bool is_gray = IsGray(obj, &fake_address_dependency); + if (kEnableReadBarrierInvariantChecks) { + CHECK_EQ(fake_address_dependency, 0U) << obj << " rb_state=" << obj->GetReadBarrierState(); + } ref_addr = reinterpret_cast<mirror::HeapReference<MirrorType>*>( - rb_ptr_high_bits | reinterpret_cast<uintptr_t>(ref_addr)); + fake_address_dependency | reinterpret_cast<uintptr_t>(ref_addr)); MirrorType* ref = ref_addr->AsMirrorPtr(); MirrorType* old_ref = ref; if (is_gray) { @@ -60,9 +62,6 @@ inline MirrorType* ReadBarrier::Barrier( offset, old_ref, ref); } } - if (kEnableReadBarrierInvariantChecks) { - CHECK_EQ(rb_ptr_high_bits, 0U) << obj << " rb_ptr=" << obj->GetReadBarrierPointer(); - } AssertToSpaceInvariant(obj, offset, ref); return ref; } else if (kUseBrooksReadBarrier) { @@ -223,20 +222,14 @@ inline mirror::Object* ReadBarrier::Mark(mirror::Object* obj) { return Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->MarkFromReadBarrier(obj); } -inline bool ReadBarrier::HasGrayReadBarrierPointer(mirror::Object* obj, - uintptr_t* out_rb_ptr_high_bits) { - mirror::Object* rb_ptr = obj->GetReadBarrierPointer(); - uintptr_t rb_ptr_bits = reinterpret_cast<uintptr_t>(rb_ptr); - uintptr_t rb_ptr_low_bits = rb_ptr_bits & rb_ptr_mask_; - if (kEnableReadBarrierInvariantChecks) { - CHECK(rb_ptr_low_bits == white_ptr_ || rb_ptr_low_bits == gray_ptr_ || - rb_ptr_low_bits == black_ptr_) - << "obj=" << obj << " rb_ptr=" << rb_ptr << " " << obj->PrettyTypeOf(); - } - bool is_gray = rb_ptr_low_bits == gray_ptr_; - // The high bits are supposed to be zero. We check this on the caller side. - *out_rb_ptr_high_bits = rb_ptr_bits & ~rb_ptr_mask_; - return is_gray; +inline bool ReadBarrier::IsGray(mirror::Object* obj, uintptr_t* fake_address_dependency) { + return obj->GetReadBarrierState(fake_address_dependency) == gray_state_; +} + +inline bool ReadBarrier::IsGray(mirror::Object* obj) { + // Use a load-acquire to load the read barrier bit to avoid reordering with the subsequent load. + // GetReadBarrierStateAcquire() has load-acquire semantics. + return obj->GetReadBarrierStateAcquire() == gray_state_; } } // namespace art diff --git a/runtime/read_barrier.h b/runtime/read_barrier.h index a8618614db..cbc26977fb 100644 --- a/runtime/read_barrier.h +++ b/runtime/read_barrier.h @@ -82,26 +82,32 @@ class ReadBarrier { // ALWAYS_INLINE on this caused a performance regression b/26744236. static mirror::Object* Mark(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_); - static mirror::Object* WhitePtr() { - return reinterpret_cast<mirror::Object*>(white_ptr_); + static constexpr uint32_t WhiteState() { + return white_state_; } - static mirror::Object* GrayPtr() { - return reinterpret_cast<mirror::Object*>(gray_ptr_); - } - static mirror::Object* BlackPtr() { - return reinterpret_cast<mirror::Object*>(black_ptr_); + static constexpr uint32_t GrayState() { + return gray_state_; } - ALWAYS_INLINE static bool HasGrayReadBarrierPointer(mirror::Object* obj, - uintptr_t* out_rb_ptr_high_bits) + // fake_address_dependency will be zero which should be bitwise-or'ed with the address of the + // subsequent load to prevent the reordering of the read barrier bit load and the subsequent + // object reference load (from one of `obj`'s fields). + // *fake_address_dependency will be set to 0. + ALWAYS_INLINE static bool IsGray(mirror::Object* obj, uintptr_t* fake_address_dependency) + REQUIRES_SHARED(Locks::mutator_lock_); + + // This uses a load-acquire to load the read barrier bit internally to prevent the reordering of + // the read barrier bit load and the subsequent load. + ALWAYS_INLINE static bool IsGray(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_); - // Note: These couldn't be constexpr pointers as reinterpret_cast isn't compatible with them. - static constexpr uintptr_t white_ptr_ = 0x0; // Not marked. - static constexpr uintptr_t gray_ptr_ = 0x1; // Marked, but not marked through. On mark stack. - // TODO: black_ptr_ is unused, we should remove it. - static constexpr uintptr_t black_ptr_ = 0x2; // Marked through. Used for non-moving objects. - static constexpr uintptr_t rb_ptr_mask_ = 0x1; // The low bits for white|gray. + static bool IsValidReadBarrierState(uint32_t rb_state) { + return rb_state == white_state_ || rb_state == gray_state_; + } + + static constexpr uint32_t white_state_ = 0x0; // Not marked. + static constexpr uint32_t gray_state_ = 0x1; // Marked, but not marked through. On mark stack. + static constexpr uint32_t rb_state_mask_ = 0x1; // The low bits for white|gray. }; } // namespace art diff --git a/runtime/reflection-inl.h b/runtime/reflection-inl.h index c4d4fae17c..68e7a10e01 100644 --- a/runtime/reflection-inl.h +++ b/runtime/reflection-inl.h @@ -29,11 +29,10 @@ namespace art { -inline bool ConvertPrimitiveValue(bool unbox_for_result, - Primitive::Type srcType, - Primitive::Type dstType, - const JValue& src, - JValue* dst) { +inline bool ConvertPrimitiveValueNoThrow(Primitive::Type srcType, + Primitive::Type dstType, + const JValue& src, + JValue* dst) { DCHECK(srcType != Primitive::kPrimNot && dstType != Primitive::kPrimNot); if (LIKELY(srcType == dstType)) { dst->SetJ(src.GetJ()); @@ -91,6 +90,18 @@ inline bool ConvertPrimitiveValue(bool unbox_for_result, default: break; } + return false; +} + +inline bool ConvertPrimitiveValue(bool unbox_for_result, + Primitive::Type srcType, + Primitive::Type dstType, + const JValue& src, + JValue* dst) { + if (ConvertPrimitiveValueNoThrow(srcType, dstType, src, dst)) { + return true; + } + if (!unbox_for_result) { ThrowIllegalArgumentException(StringPrintf("Invalid primitive conversion from %s to %s", PrettyDescriptor(srcType).c_str(), diff --git a/runtime/reflection.h b/runtime/reflection.h index 6e5ef712a7..f2652fd4b6 100644 --- a/runtime/reflection.h +++ b/runtime/reflection.h @@ -47,6 +47,12 @@ bool UnboxPrimitiveForResult(ObjPtr<mirror::Object> o, JValue* unboxed_value) REQUIRES_SHARED(Locks::mutator_lock_); +ALWAYS_INLINE bool ConvertPrimitiveValueNoThrow(Primitive::Type src_class, + Primitive::Type dst_class, + const JValue& src, + JValue* dst) + REQUIRES_SHARED(Locks::mutator_lock_); + ALWAYS_INLINE bool ConvertPrimitiveValue(bool unbox_for_result, Primitive::Type src_class, Primitive::Type dst_class, diff --git a/runtime/runtime.h b/runtime/runtime.h index 77ec23812a..6abe682d4b 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -182,7 +182,7 @@ class Runtime { return compiler_options_; } - void AddCompilerOption(std::string option) { + void AddCompilerOption(const std::string& option) { compiler_options_.push_back(option); } @@ -326,6 +326,7 @@ class Runtime { // Visit all the roots. If only_dirty is true then non-dirty roots won't be visited. If // clean_dirty is true then dirty roots will be marked as non-dirty after visiting. void VisitRoots(RootVisitor* visitor, VisitRootFlags flags = kVisitRootFlagAllRoots) + REQUIRES(!Locks::classlinker_classes_lock_, !Locks::trace_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // Visit image roots, only used for hprof since the GC uses the image space mod union table @@ -335,6 +336,7 @@ class Runtime { // Visit all of the roots we can do safely do concurrently. void VisitConcurrentRoots(RootVisitor* visitor, VisitRootFlags flags = kVisitRootFlagAllRoots) + REQUIRES(!Locks::classlinker_classes_lock_, !Locks::trace_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // Visit all of the non thread roots, we can do this with mutators unpaused. diff --git a/runtime/thread.h b/runtime/thread.h index 75b5b123da..4f26803726 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -1567,7 +1567,8 @@ class Thread { class SCOPED_CAPABILITY ScopedAssertNoThreadSuspension { public: - ALWAYS_INLINE ScopedAssertNoThreadSuspension(const char* cause) ACQUIRE(Roles::uninterruptible_) { + ALWAYS_INLINE explicit ScopedAssertNoThreadSuspension(const char* cause) + ACQUIRE(Roles::uninterruptible_) { if (kIsDebugBuild) { self_ = Thread::Current(); old_cause_ = self_->StartAssertNoThreadSuspension(cause); diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc index 97bc79cfcc..d9e3ea7a0d 100644 --- a/runtime/verifier/method_verifier.cc +++ b/runtime/verifier/method_verifier.cc @@ -956,7 +956,7 @@ void MethodVerifier::PrependToLastFailMessage(std::string prepend) { delete last_fail_message; } -void MethodVerifier::AppendToLastFailMessage(std::string append) { +void MethodVerifier::AppendToLastFailMessage(const std::string& append) { size_t failure_num = failure_messages_.size(); DCHECK_NE(failure_num, 0U); std::ostringstream* last_fail_message = failure_messages_[failure_num - 1]; diff --git a/runtime/verifier/method_verifier.h b/runtime/verifier/method_verifier.h index eb8b7a639d..c6ce583ab4 100644 --- a/runtime/verifier/method_verifier.h +++ b/runtime/verifier/method_verifier.h @@ -318,7 +318,7 @@ class MethodVerifier { void PrependToLastFailMessage(std::string); // Adds the given string to the end of the last failure message. - void AppendToLastFailMessage(std::string); + void AppendToLastFailMessage(const std::string& append); // Verification result for method(s). Includes a (maximum) failure kind, and (the union of) // all failure types. diff --git a/runtime/verifier/register_line.cc b/runtime/verifier/register_line.cc index 4ec2da6e5a..da3d946142 100644 --- a/runtime/verifier/register_line.cc +++ b/runtime/verifier/register_line.cc @@ -411,7 +411,7 @@ bool FindLockAliasedRegister(uint32_t src, } // Scan the map for the same value. - for (const std::pair<uint32_t, uint32_t>& pair : search_map) { + for (const std::pair<const uint32_t, uint32_t>& pair : search_map) { if (pair.first != src && pair.second == src_lock_levels) { return true; } diff --git a/runtime/verifier/verifier_deps.cc b/runtime/verifier/verifier_deps.cc index bdf63cb45b..c395612d72 100644 --- a/runtime/verifier/verifier_deps.cc +++ b/runtime/verifier/verifier_deps.cc @@ -607,7 +607,8 @@ void VerifierDeps::Dump(VariableIndentationOutputStream* vios) const { } } -bool VerifierDeps::Verify(Handle<mirror::ClassLoader> class_loader, Thread* self) const { +bool VerifierDeps::ValidateDependencies(Handle<mirror::ClassLoader> class_loader, + Thread* self) const { for (const auto& entry : dex_deps_) { if (!VerifyDexFile(class_loader, *entry.first, *entry.second, self)) { return false; diff --git a/runtime/verifier/verifier_deps.h b/runtime/verifier/verifier_deps.h index eea0299f6b..7b419d4260 100644 --- a/runtime/verifier/verifier_deps.h +++ b/runtime/verifier/verifier_deps.h @@ -109,11 +109,18 @@ class VerifierDeps { void Dump(VariableIndentationOutputStream* vios) const NO_THREAD_SAFETY_ANALYSIS; - // Verify the encoded dependencies of this `VerifierDeps`. + // Verify the encoded dependencies of this `VerifierDeps` are still valid. // NO_THREAD_SAFETY_ANALYSIS, as this must be called on a read-only `VerifierDeps`. - bool Verify(Handle<mirror::ClassLoader> class_loader, Thread* self) const + bool ValidateDependencies(Handle<mirror::ClassLoader> class_loader, Thread* self) const NO_THREAD_SAFETY_ANALYSIS; + // NO_THREAD_SAFETY_ANALSYS, as this is queried when the VerifierDeps are + // fully created. + const std::vector<uint16_t>& GetUnverifiedClasses(const DexFile& dex_file) const + NO_THREAD_SAFETY_ANALYSIS { + return GetDexFileDeps(dex_file)->unverified_classes_; + } + private: static constexpr uint16_t kUnresolvedMarker = static_cast<uint16_t>(-1); @@ -317,6 +324,7 @@ class VerifierDeps { ART_FRIEND_TEST(VerifierDepsTest, EncodeDecode); ART_FRIEND_TEST(VerifierDepsTest, EncodeDecodeMulti); ART_FRIEND_TEST(VerifierDepsTest, VerifyDeps); + ART_FRIEND_TEST(VerifierDepsTest, CompilerDriver); }; } // namespace verifier diff --git a/test/115-native-bridge/nativebridge.cc b/test/115-native-bridge/nativebridge.cc index 56d737f4ee..41329af138 100644 --- a/test/115-native-bridge/nativebridge.cc +++ b/test/115-native-bridge/nativebridge.cc @@ -466,7 +466,7 @@ extern "C" int native_bridge_unloadLibrary(void* handle ATTRIBUTE_UNUSED) { return 0; } -extern "C" char* native_bridge_getError() { +extern "C" const char* native_bridge_getError() { printf("dlerror() in native bridge.\n"); return nullptr; } diff --git a/test/527-checker-array-access-split/info.txt b/test/527-checker-array-access-split/info.txt index 920680462d..a39bea37cf 100644 --- a/test/527-checker-array-access-split/info.txt +++ b/test/527-checker-array-access-split/info.txt @@ -1 +1 @@ -Test arm64-specific array access optimization. +Test arm- and arm64-specific array access optimization. diff --git a/test/530-checker-lse/src/Main.java b/test/530-checker-lse/src/Main.java index 6b0dedfe9d..a61b9a0c06 100644 --- a/test/530-checker-lse/src/Main.java +++ b/test/530-checker-lse/src/Main.java @@ -744,6 +744,20 @@ public class Main { return 1.0f; } + /// CHECK-START: double Main.getCircleArea(double, boolean) load_store_elimination (before) + /// CHECK: NewInstance + + /// CHECK-START: double Main.getCircleArea(double, boolean) load_store_elimination (after) + /// CHECK-NOT: NewInstance + + private static double getCircleArea(double radius, boolean b) { + double area = 0d; + if (b) { + area = new Circle(radius).getArea(); + } + return area; + } + static void assertIntEquals(int result, int expected) { if (expected != result) { throw new Error("Expected: " + expected + ", found: " + result); @@ -808,6 +822,8 @@ public class Main { assertIntEquals(sumWithinRange(array, 1, 5), 11); assertFloatEquals(testAllocationEliminationWithLoops(), 1.0f); assertFloatEquals(mF, 0f); + assertDoubleEquals(Math.PI * Math.PI * Math.PI, getCircleArea(Math.PI, true)); + assertDoubleEquals(0d, getCircleArea(Math.PI, false)); } static boolean sFlag; diff --git a/test/562-no-intermediate/expected.txt b/test/562-checker-no-intermediate/expected.txt index e69de29bb2..e69de29bb2 100644 --- a/test/562-no-intermediate/expected.txt +++ b/test/562-checker-no-intermediate/expected.txt diff --git a/test/562-no-intermediate/info.txt b/test/562-checker-no-intermediate/info.txt index 4f21aebd03..38f1f65dd6 100644 --- a/test/562-no-intermediate/info.txt +++ b/test/562-checker-no-intermediate/info.txt @@ -1,2 +1,2 @@ Regression test for optimizing, checking that there is no -intermediate address between a Java call. +intermediate address live across a Java call. diff --git a/test/562-checker-no-intermediate/src/Main.java b/test/562-checker-no-intermediate/src/Main.java new file mode 100644 index 0000000000..104ba8bc06 --- /dev/null +++ b/test/562-checker-no-intermediate/src/Main.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + + /** + * Check that the intermediate address computation is not reordered or merged + * across the call to Math.abs(). + */ + + /// CHECK-START-ARM: void Main.main(java.lang.String[]) instruction_simplifier_arm (before) + /// CHECK-DAG: <<ConstM42:i\d+>> IntConstant -42 + /// CHECK-DAG: <<Array:l\d+>> NullCheck + /// CHECK-DAG: <<Index:i\d+>> BoundsCheck + /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Array>>,<<Index>>] + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] + /// CHECK-DAG: ArraySet [<<Array>>,<<Index>>,<<Add>>] + + /// CHECK-START-ARM: void Main.main(java.lang.String[]) instruction_simplifier_arm (after) + /// CHECK-DAG: <<ConstM42:i\d+>> IntConstant -42 + /// CHECK-DAG: <<DataOffset:i\d+>> IntConstant + /// CHECK-DAG: <<Array:l\d+>> NullCheck + /// CHECK-DAG: <<Index:i\d+>> BoundsCheck + /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] + /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] + + /// CHECK-START-ARM: void Main.main(java.lang.String[]) GVN$after_arch (after) + /// CHECK-DAG: <<ConstM42:i\d+>> IntConstant -42 + /// CHECK-DAG: <<DataOffset:i\d+>> IntConstant + /// CHECK-DAG: <<Array:l\d+>> NullCheck + /// CHECK-DAG: <<Index:i\d+>> BoundsCheck + /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] + /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] + + + /// CHECK-START-ARM64: void Main.main(java.lang.String[]) instruction_simplifier_arm64 (before) + /// CHECK-DAG: <<ConstM42:i\d+>> IntConstant -42 + /// CHECK-DAG: <<Array:l\d+>> NullCheck + /// CHECK-DAG: <<Index:i\d+>> BoundsCheck + /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Array>>,<<Index>>] + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] + /// CHECK-DAG: ArraySet [<<Array>>,<<Index>>,<<Add>>] + + /// CHECK-START-ARM64: void Main.main(java.lang.String[]) instruction_simplifier_arm64 (after) + /// CHECK-DAG: <<ConstM42:i\d+>> IntConstant -42 + /// CHECK-DAG: <<DataOffset:i\d+>> IntConstant + /// CHECK-DAG: <<Array:l\d+>> NullCheck + /// CHECK-DAG: <<Index:i\d+>> BoundsCheck + /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] + /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] + + /// CHECK-START-ARM64: void Main.main(java.lang.String[]) GVN$after_arch (after) + /// CHECK-DAG: <<ConstM42:i\d+>> IntConstant -42 + /// CHECK-DAG: <<DataOffset:i\d+>> IntConstant + /// CHECK-DAG: <<Array:l\d+>> NullCheck + /// CHECK-DAG: <<Index:i\d+>> BoundsCheck + /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] + /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] + /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] + + public static void main(String[] args) { + array[index] += Math.abs(-42); + } + + static int index = 0; + static int[] array = new int[2]; +} diff --git a/test/562-no-intermediate/src/Main.java b/test/562-no-intermediate/src/Main.java deleted file mode 100644 index 3b74d6f269..0000000000 --- a/test/562-no-intermediate/src/Main.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class Main { - - /// CHECK-START-ARM64: int Main.main(String[]) register_allocator (after) - /// CHECK-NOT: IntermediateAddress - public static void main(String[] args) { - array[index] += Math.cos(42); - } - - static int index = 0; - static double[] array = new double[2]; -} diff --git a/test/618-checker-induction/src/Main.java b/test/618-checker-induction/src/Main.java index d8bc611c22..f85479aa54 100644 --- a/test/618-checker-induction/src/Main.java +++ b/test/618-checker-induction/src/Main.java @@ -92,6 +92,43 @@ public class Main { } } + /// CHECK-START: void Main.deadConditional(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:{{B\d+}} outer_loop:none + // + /// CHECK-START: void Main.deadConditional(int) loop_optimization (after) + /// CHECK-NOT: Phi loop:{{B\d+}} + public static void deadConditional(int n) { + int k = 0; + int m = 0; + for (int i = 0; i < n; i++) { + if (i == 3) + k = i; + else + m = i; + } + } + + /// CHECK-START: void Main.deadConditionalCycle(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + // + /// CHECK-START: void Main.deadConditionalCycle(int) loop_optimization (after) + /// CHECK-NOT: Phi loop:{{B\d+}} + public static void deadConditionalCycle(int n) { + int k = 0; + int m = 0; + for (int i = 0; i < n; i++) { + if (i == 3) + k--; + else + m++; + } + } + + /// CHECK-START: void Main.deadInduction() loop_optimization (before) /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none @@ -668,6 +705,8 @@ public class Main { potentialInfiniteLoop(4); deadNestedLoops(); deadNestedAndFollowingLoops(); + deadConditional(4); + deadConditionalCycle(4); deadInduction(); for (int i = 0; i < a.length; i++) { diff --git a/test/622-checker-bce-regressions/expected.txt b/test/622-checker-bce-regressions/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/622-checker-bce-regressions/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/622-checker-bce-regressions/info.txt b/test/622-checker-bce-regressions/info.txt new file mode 100644 index 0000000000..a753dfa2ff --- /dev/null +++ b/test/622-checker-bce-regressions/info.txt @@ -0,0 +1 @@ +Regression tests on BCE. diff --git a/test/622-checker-bce-regressions/src/Main.java b/test/622-checker-bce-regressions/src/Main.java new file mode 100644 index 0000000000..6ba2644b97 --- /dev/null +++ b/test/622-checker-bce-regressions/src/Main.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Regression tests for BCE. + */ +public class Main { + + static int[] array = new int[10]; + + /// CHECK-START: int Main.doNotVisitAfterForwardBCE(int[]) BCE (before) + /// CHECK-DAG: BoundsCheck loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: BoundsCheck loop:<<Loop>> outer_loop:none + // + /// CHECK-START: int Main.doNotVisitAfterForwardBCE(int[]) BCE (after) + /// CHECK-NOT: BoundsCheck + static int doNotVisitAfterForwardBCE(int[] a) { + if (a == null) { + throw new Error("Null"); + } + int k = 0; + int j = 0; + for (int i = 1; i < 10; i++) { + j = i - 1; + // b/32547652: after DCE, bounds checks become consecutive, + // and second should not be revisited after forward BCE. + k = a[i] + a[i - 1]; + } + return j; + } + + public static void main(String[] args) { + expectEquals(8, doNotVisitAfterForwardBCE(array)); + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } +} diff --git a/test/623-checker-loop-regressions/expected.txt b/test/623-checker-loop-regressions/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/623-checker-loop-regressions/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/623-checker-loop-regressions/info.txt b/test/623-checker-loop-regressions/info.txt new file mode 100644 index 0000000000..6271600432 --- /dev/null +++ b/test/623-checker-loop-regressions/info.txt @@ -0,0 +1 @@ +Regression tests on loop optimizations. diff --git a/test/623-checker-loop-regressions/src/Main.java b/test/623-checker-loop-regressions/src/Main.java new file mode 100644 index 0000000000..ce5bda1393 --- /dev/null +++ b/test/623-checker-loop-regressions/src/Main.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Regression tests for loop optimizations. + */ +public class Main { + + /// CHECK-START: int Main.earlyExitFirst(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + // + /// CHECK-START: int Main.earlyExitFirst(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + static int earlyExitFirst(int m) { + int k = 0; + for (int i = 0; i < 10; i++) { + if (i == m) { + return k; + } + k++; + } + return k; + } + + /// CHECK-START: int Main.earlyExitLast(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + // + /// CHECK-START: int Main.earlyExitLast(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop>> outer_loop:none + static int earlyExitLast(int m) { + int k = 0; + for (int i = 0; i < 10; i++) { + k++; + if (i == m) { + return k; + } + } + return k; + } + + /// CHECK-START: int Main.earlyExitNested() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop2:B\d+>> outer_loop:<<Loop1>> + /// CHECK-DAG: Phi loop:<<Loop2>> outer_loop:<<Loop1>> + // + /// CHECK-START: int Main.earlyExitNested() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop1>> outer_loop:none + // + /// CHECK-START: int Main.earlyExitNested() loop_optimization (after) + /// CHECK-NOT: Phi loop:{{B\d+}} outer_loop:{{B\d+}} + static int earlyExitNested() { + int offset = 0; + for (int i = 0; i < 2; i++) { + int start = offset; + // This loop can be removed. + for (int j = 0; j < 2; j++) { + offset++; + } + if (i == 1) { + return start; + } + } + return 0; + } + + public static void main(String[] args) { + expectEquals(10, earlyExitFirst(-1)); + for (int i = 0; i <= 10; i++) { + expectEquals(i, earlyExitFirst(i)); + } + expectEquals(10, earlyExitFirst(11)); + + expectEquals(10, earlyExitLast(-1)); + for (int i = 0; i < 10; i++) { + expectEquals(i + 1, earlyExitLast(i)); + } + expectEquals(10, earlyExitLast(10)); + expectEquals(10, earlyExitLast(11)); + + expectEquals(2, earlyExitNested()); + + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } +} diff --git a/test/955-methodhandles-smali/expected.txt b/test/955-methodhandles-smali/expected.txt index 047a287eb8..5de1274e37 100644 --- a/test/955-methodhandles-smali/expected.txt +++ b/test/955-methodhandles-smali/expected.txt @@ -5,4 +5,5 @@ 40 43 44 -0-11 +0 +-1 diff --git a/test/955-methodhandles-smali/smali/Main.smali b/test/955-methodhandles-smali/smali/Main.smali index 9681d56f00..52460a8985 100644 --- a/test/955-methodhandles-smali/smali/Main.smali +++ b/test/955-methodhandles-smali/smali/Main.smali @@ -220,24 +220,22 @@ invoke-polymorphic {v0, v1, v1}, Ljava/lang/invoke/MethodHandle;->invoke([Ljava/lang/Object;)Ljava/lang/Object;, (Ljava/lang/Long;Ljava/lang/Long;)I move-result v3 sget-object v4, Ljava/lang/System;->out:Ljava/io/PrintStream; - invoke-virtual {v4, v3}, Ljava/io/PrintStream;->print(I)V + invoke-virtual {v4, v3}, Ljava/io/PrintStream;->println(I)V # Call compareTo(long) - this is an implicit box. const-wide v2, 44 invoke-polymorphic {v0, v1, v2, v3}, Ljava/lang/invoke/MethodHandle;->invoke([Ljava/lang/Object;)Ljava/lang/Object;, (Ljava/lang/Long;J)I move-result v3 sget-object v4, Ljava/lang/System;->out:Ljava/io/PrintStream; - invoke-virtual {v4, v3}, Ljava/io/PrintStream;->print(I)V + invoke-virtual {v4, v3}, Ljava/io/PrintStream;->println(I)V # Call compareTo(int) - this is an implicit box. - const v2, 40 - invoke-polymorphic {v0, v1, v2}, Ljava/lang/invoke/MethodHandle;->invoke([Ljava/lang/Object;)Ljava/lang/Object;, (Ljava/lang/Long;I)I - move-result v3 - sget-object v4, Ljava/lang/System;->out:Ljava/io/PrintStream; - invoke-virtual {v4, v3}, Ljava/io/PrintStream;->print(I)V - - # Add a newline at the end of file. - invoke-virtual {v4}, Ljava/io/PrintStream;->println()V +# This throws WrongMethodTypeException as it's a two step conversion int->long->Long or int->Integer->Long. +# const v2, 40 +# invoke-polymorphic {v0, v1, v2}, Ljava/lang/invoke/MethodHandle;->invoke([Ljava/lang/Object;)Ljava/lang/Object;, (Ljava/lang/Long;I)I +# move-result v3 +# sget-object v4, Ljava/lang/System;->out:Ljava/io/PrintStream; +# invoke-virtual {v4, v3}, Ljava/io/PrintStream;->print(I)V return-void .end method diff --git a/test/956-methodhandles/src/Main.java b/test/956-methodhandles/src/Main.java index 44c0447771..780513f9ed 100644 --- a/test/956-methodhandles/src/Main.java +++ b/test/956-methodhandles/src/Main.java @@ -20,6 +20,10 @@ import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; import java.lang.invoke.WrongMethodTypeException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; + public class Main { public static class A { @@ -59,6 +63,8 @@ public class Main { testfindSpecial_invokeDirectBehaviour(); testExceptionDetailMessages(); testfindVirtual(); + testUnreflects(); + testAsType(); } public static void testfindSpecial_invokeSuperBehaviour() throws Throwable { @@ -262,6 +268,231 @@ public class Main { System.out.println("Unexpected return value for BarImpl#superPackageMethod: " + str); } } + + static class UnreflectTester { + public String publicField; + private String privateField; + + public static String publicStaticField = "publicStaticValue"; + private static String privateStaticField = "privateStaticValue"; + + private UnreflectTester(String val) { + publicField = val; + privateField = val; + } + + // NOTE: The boolean constructor argument only exists to give this a + // different signature. + public UnreflectTester(String val, boolean unused) { + this(val); + } + + private static String privateStaticMethod() { + return "privateStaticMethod"; + } + + private String privateMethod() { + return "privateMethod"; + } + + public static String publicStaticMethod() { + return "publicStaticMethod"; + } + + public String publicMethod() { + return "publicMethod"; + } + } + + public static void testUnreflects() throws Throwable { + UnreflectTester instance = new UnreflectTester("unused"); + Method publicMethod = UnreflectTester.class.getMethod("publicMethod"); + + MethodHandle mh = MethodHandles.lookup().unreflect(publicMethod); + assertEquals("publicMethod", (String) mh.invoke(instance)); + assertEquals("publicMethod", (String) mh.invokeExact(instance)); + + Method publicStaticMethod = UnreflectTester.class.getMethod("publicStaticMethod"); + mh = MethodHandles.lookup().unreflect(publicStaticMethod); + assertEquals("publicStaticMethod", (String) mh.invoke()); + assertEquals("publicStaticMethod", (String) mh.invokeExact()); + + Method privateMethod = UnreflectTester.class.getDeclaredMethod("privateMethod"); + try { + mh = MethodHandles.lookup().unreflect(privateMethod); + fail(); + } catch (IllegalAccessException expected) {} + + privateMethod.setAccessible(true); + mh = MethodHandles.lookup().unreflect(privateMethod); + assertEquals("privateMethod", (String) mh.invoke(instance)); + assertEquals("privateMethod", (String) mh.invokeExact(instance)); + + Method privateStaticMethod = UnreflectTester.class.getDeclaredMethod("privateStaticMethod"); + try { + mh = MethodHandles.lookup().unreflect(privateStaticMethod); + fail(); + } catch (IllegalAccessException expected) {} + + privateStaticMethod.setAccessible(true); + mh = MethodHandles.lookup().unreflect(privateStaticMethod); + assertEquals("privateStaticMethod", (String) mh.invoke()); + assertEquals("privateStaticMethod", (String) mh.invokeExact()); + + Constructor privateConstructor = UnreflectTester.class.getDeclaredConstructor(String.class); + try { + mh = MethodHandles.lookup().unreflectConstructor(privateConstructor); + fail(); + } catch (IllegalAccessException expected) {} + + privateConstructor.setAccessible(true); + mh = MethodHandles.lookup().unreflectConstructor(privateConstructor); + // TODO(narayan): Method handle constructor invokes are not supported yet. + // + // UnreflectTester tester = (UnreflectTester) mh.invoke("foo"); + // UnreflectTester tester = (UnreflectTester) mh.invoke("fooExact"); + + Constructor publicConstructor = UnreflectTester.class.getConstructor(String.class, + boolean.class); + mh = MethodHandles.lookup().unreflectConstructor(publicConstructor); + // TODO(narayan): Method handle constructor invokes are not supported yet. + // + // UnreflectTester tester = (UnreflectTester) mh.invoke("foo"); + // UnreflectTester tester = (UnreflectTester) mh.invoke("fooExact"); + + // TODO(narayan): Non exact invokes for field sets/gets are not implemented yet. + // + // assertEquals("instanceValue", (String) mh.invoke(new UnreflectTester("instanceValue"))); + Field publicField = UnreflectTester.class.getField("publicField"); + mh = MethodHandles.lookup().unreflectGetter(publicField); + instance = new UnreflectTester("instanceValue"); + assertEquals("instanceValue", (String) mh.invokeExact(instance)); + + mh = MethodHandles.lookup().unreflectSetter(publicField); + instance = new UnreflectTester("instanceValue"); + mh.invokeExact(instance, "updatedInstanceValue"); + assertEquals("updatedInstanceValue", instance.publicField); + + Field publicStaticField = UnreflectTester.class.getField("publicStaticField"); + mh = MethodHandles.lookup().unreflectGetter(publicStaticField); + UnreflectTester.publicStaticField = "updatedStaticValue"; + assertEquals("updatedStaticValue", (String) mh.invokeExact()); + + mh = MethodHandles.lookup().unreflectSetter(publicStaticField); + UnreflectTester.publicStaticField = "updatedStaticValue"; + mh.invokeExact("updatedStaticValue2"); + assertEquals("updatedStaticValue2", UnreflectTester.publicStaticField); + + Field privateField = UnreflectTester.class.getDeclaredField("privateField"); + try { + mh = MethodHandles.lookup().unreflectGetter(privateField); + fail(); + } catch (IllegalAccessException expected) { + } + try { + mh = MethodHandles.lookup().unreflectSetter(privateField); + fail(); + } catch (IllegalAccessException expected) { + } + + privateField.setAccessible(true); + + mh = MethodHandles.lookup().unreflectGetter(privateField); + instance = new UnreflectTester("instanceValue"); + assertEquals("instanceValue", (String) mh.invokeExact(instance)); + + mh = MethodHandles.lookup().unreflectSetter(privateField); + instance = new UnreflectTester("instanceValue"); + mh.invokeExact(instance, "updatedInstanceValue"); + assertEquals("updatedInstanceValue", instance.privateField); + + Field privateStaticField = UnreflectTester.class.getDeclaredField("privateStaticField"); + try { + mh = MethodHandles.lookup().unreflectGetter(privateStaticField); + fail(); + } catch (IllegalAccessException expected) { + } + try { + mh = MethodHandles.lookup().unreflectSetter(privateStaticField); + fail(); + } catch (IllegalAccessException expected) { + } + + privateStaticField.setAccessible(true); + mh = MethodHandles.lookup().unreflectGetter(privateStaticField); + privateStaticField.set(null, "updatedStaticValue"); + assertEquals("updatedStaticValue", (String) mh.invokeExact()); + + mh = MethodHandles.lookup().unreflectSetter(privateStaticField); + privateStaticField.set(null, "updatedStaticValue"); + mh.invokeExact("updatedStaticValue2"); + assertEquals("updatedStaticValue2", (String) privateStaticField.get(null)); + } + + // This method only exists to fool Jack's handling of types. See b/32536744. + public static CharSequence getSequence() { + return "foo"; + } + + public static void testAsType() throws Throwable { + // The type of this handle is (String, String)String. + MethodHandle mh = MethodHandles.lookup().findVirtual(String.class, + "concat", MethodType.methodType(String.class, String.class)); + + // Change it to (CharSequence, String)Object. + MethodHandle asType = mh.asType( + MethodType.methodType(Object.class, CharSequence.class, String.class)); + + Object obj = asType.invokeExact((CharSequence) getSequence(), "bar"); + assertEquals("foobar", (String) obj); + + // Should fail due to a wrong return type. + try { + String str = (String) asType.invokeExact((CharSequence) getSequence(), "bar"); + fail(); + } catch (WrongMethodTypeException expected) { + } + + // Should fail due to a wrong argument type (String instead of Charsequence). + try { + String str = (String) asType.invokeExact("baz", "bar"); + fail(); + } catch (WrongMethodTypeException expected) { + } + + // Calls to asType should fail if the types are not convertible. + // + // Bad return type conversion. + try { + mh.asType(MethodType.methodType(int.class, String.class, String.class)); + fail(); + } catch (WrongMethodTypeException expected) { + } + + // Bad argument conversion. + try { + mh.asType(MethodType.methodType(String.class, int.class, String.class)); + fail(); + } catch (WrongMethodTypeException expected) { + } + } + + public static void assertEquals(String s1, String s2) { + if (s1 == s2) { + return; + } + + if (s1 != null && s2 != null && s1.equals(s2)) { + return; + } + + throw new AssertionError("assertEquals s1: " + s1 + ", s2: " + s2); + } + + public static void fail() { + System.out.println("fail"); + Thread.dumpStack(); + } } diff --git a/test/957-methodhandle-transforms/expected.txt b/test/957-methodhandle-transforms/expected.txt index fc4fdd6070..7540ef74f3 100644 --- a/test/957-methodhandle-transforms/expected.txt +++ b/test/957-methodhandle-transforms/expected.txt @@ -4,6 +4,7 @@ Message: foo, Message2: 42 Message: foo, Message2: 42 Message: foo, Message2: 42 Message: foo, Message2: 42 +Message: foo, Message2: 42 Target: Arg1: foo, Arg2: 42 Target: Arg1: foo, Arg2: 42 Handler: java.lang.IllegalArgumentException: exceptionMessage, Arg1: foo, Arg2: 42, ExMsg: exceptionMessage @@ -14,3 +15,4 @@ target: target, 42, 56 target: target, 42, 56 fallback: fallback, 42, 56 target: target, 42, 56 +target: target, 42, 56 diff --git a/test/957-methodhandle-transforms/src/Main.java b/test/957-methodhandle-transforms/src/Main.java index 4a27086924..6c977f4f34 100644 --- a/test/957-methodhandle-transforms/src/Main.java +++ b/test/957-methodhandle-transforms/src/Main.java @@ -26,6 +26,10 @@ public class Main { testDropArguments(); testCatchException(); testGuardWithTest(); + testArrayElementGetter(); + testArrayElementSetter(); + testIdentity(); + testConstant(); } public static void testThrowException() throws Throwable { @@ -90,6 +94,12 @@ public class Main { // it's IAE and should be WMTE instead. } + // Check that asType works as expected. + transform = MethodHandles.dropArguments(delegate, 0, int.class, Object.class); + transform = transform.asType(MethodType.methodType(void.class, + new Class<?>[] { short.class, Object.class, String.class, long.class })); + transform.invokeExact((short) 45, new Object(), "foo", 42l); + // Invalid argument location, should not be allowed. try { MethodHandles.dropArguments(delegate, -1, int.class, Object.class); @@ -183,6 +193,14 @@ public class Main { assertEquals("java.lang.IllegalArgumentException: exceptionMessage", returnVal); returnVal = (String) adapter.invokeExact("foo", 42l, "exceptionMessage2"); assertEquals("java.lang.IllegalArgumentException: exceptionMessage2", returnVal); + + // Check that asType works as expected. + adapter = MethodHandles.catchException(target, IllegalArgumentException.class, + handler); + adapter = adapter.asType(MethodType.methodType(String.class, + new Class<?>[] { String.class, int.class, String.class })); + returnVal = (String) adapter.invokeExact("foo", 42, "exceptionMessage"); + assertEquals("java.lang.IllegalArgumentException: exceptionMessage", returnVal); } public static boolean testGuardWithTest_test(String arg1, long arg2) { @@ -225,6 +243,430 @@ public class Main { assertEquals("fallback", returnVal); returnVal = (String) adapter.invokeExact("target", 42l, 56); assertEquals("target", returnVal); + + // Check that asType works as expected. + adapter = adapter.asType(MethodType.methodType(String.class, + new Class<?>[] { String.class, int.class, int.class })); + returnVal = (String) adapter.invokeExact("target", 42, 56); + assertEquals("target", returnVal); + } + + public static void testArrayElementGetter() throws Throwable { + MethodHandle getter = MethodHandles.arrayElementGetter(int[].class); + + { + int[] array = new int[1]; + array[0] = 42; + int value = (int) getter.invoke(array, 0); + if (value != 42) { + System.out.println("Unexpected value: " + value); + } + + try { + value = (int) getter.invoke(array, -1); + fail(); + } catch (ArrayIndexOutOfBoundsException expected) { + } + + try { + value = (int) getter.invoke(null, -1); + fail(); + } catch (NullPointerException expected) { + } + } + + { + getter = MethodHandles.arrayElementGetter(long[].class); + long[] array = new long[1]; + array[0] = 42; + long value = (long) getter.invoke(array, 0); + if (value != 42l) { + System.out.println("Unexpected value: " + value); + } + } + + { + getter = MethodHandles.arrayElementGetter(short[].class); + short[] array = new short[1]; + array[0] = 42; + short value = (short) getter.invoke(array, 0); + if (value != 42l) { + System.out.println("Unexpected value: " + value); + } + } + + { + getter = MethodHandles.arrayElementGetter(char[].class); + char[] array = new char[1]; + array[0] = 42; + char value = (char) getter.invoke(array, 0); + if (value != 42l) { + System.out.println("Unexpected value: " + value); + } + } + + { + getter = MethodHandles.arrayElementGetter(byte[].class); + byte[] array = new byte[1]; + array[0] = (byte) 0x8; + byte value = (byte) getter.invoke(array, 0); + if (value != (byte) 0x8) { + System.out.println("Unexpected value: " + value); + } + } + + { + getter = MethodHandles.arrayElementGetter(boolean[].class); + boolean[] array = new boolean[1]; + array[0] = true; + boolean value = (boolean) getter.invoke(array, 0); + if (!value) { + System.out.println("Unexpected value: " + value); + } + } + + { + getter = MethodHandles.arrayElementGetter(float[].class); + float[] array = new float[1]; + array[0] = 42.0f; + float value = (float) getter.invoke(array, 0); + if (value != 42.0f) { + System.out.println("Unexpected value: " + value); + } + } + + { + getter = MethodHandles.arrayElementGetter(double[].class); + double[] array = new double[1]; + array[0] = 42.0; + double value = (double) getter.invoke(array, 0); + if (value != 42.0) { + System.out.println("Unexpected value: " + value); + } + } + + { + getter = MethodHandles.arrayElementGetter(String[].class); + String[] array = new String[3]; + array[0] = "42"; + array[1] = "48"; + array[2] = "54"; + String value = (String) getter.invoke(array, 0); + assertEquals("42", value); + value = (String) getter.invoke(array, 1); + assertEquals("48", value); + value = (String) getter.invoke(array, 2); + assertEquals("54", value); + } + } + + public static void testArrayElementSetter() throws Throwable { + MethodHandle setter = MethodHandles.arrayElementSetter(int[].class); + + { + int[] array = new int[2]; + setter.invoke(array, 0, 42); + setter.invoke(array, 1, 43); + + if (array[0] != 42) { + System.out.println("Unexpected value: " + array[0]); + } + if (array[1] != 43) { + System.out.println("Unexpected value: " + array[1]); + } + + try { + setter.invoke(array, -1, 42); + fail(); + } catch (ArrayIndexOutOfBoundsException expected) { + } + + try { + setter.invoke(null, 0, 42); + fail(); + } catch (NullPointerException expected) { + } + } + + { + setter = MethodHandles.arrayElementSetter(long[].class); + long[] array = new long[1]; + setter.invoke(array, 0, 42l); + if (array[0] != 42l) { + System.out.println("Unexpected value: " + array[0]); + } + } + + { + setter = MethodHandles.arrayElementSetter(short[].class); + short[] array = new short[1]; + setter.invoke(array, 0, (short) 42); + if (array[0] != 42l) { + System.out.println("Unexpected value: " + array[0]); + } + } + + { + setter = MethodHandles.arrayElementSetter(char[].class); + char[] array = new char[1]; + setter.invoke(array, 0, (char) 42); + if (array[0] != 42) { + System.out.println("Unexpected value: " + array[0]); + } + } + + { + setter = MethodHandles.arrayElementSetter(byte[].class); + byte[] array = new byte[1]; + setter.invoke(array, 0, (byte) 0x8); + if (array[0] != (byte) 0x8) { + System.out.println("Unexpected value: " + array[0]); + } + } + + { + setter = MethodHandles.arrayElementSetter(boolean[].class); + boolean[] array = new boolean[1]; + setter.invoke(array, 0, true); + if (!array[0]) { + System.out.println("Unexpected value: " + array[0]); + } + } + + { + setter = MethodHandles.arrayElementSetter(float[].class); + float[] array = new float[1]; + setter.invoke(array, 0, 42.0f); + if (array[0] != 42.0f) { + System.out.println("Unexpected value: " + array[0]); + } + } + + { + setter = MethodHandles.arrayElementSetter(double[].class); + double[] array = new double[1]; + setter.invoke(array, 0, 42.0); + if (array[0] != 42.0) { + System.out.println("Unexpected value: " + array[0]); + } + } + + { + setter = MethodHandles.arrayElementSetter(String[].class); + String[] array = new String[3]; + setter.invoke(array, 0, "42"); + setter.invoke(array, 1, "48"); + setter.invoke(array, 2, "54"); + assertEquals("42", array[0]); + assertEquals("48", array[1]); + assertEquals("54", array[2]); + } + } + + public static void testIdentity() throws Throwable { + { + MethodHandle identity = MethodHandles.identity(boolean.class); + boolean value = (boolean) identity.invoke(false); + if (value) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(byte.class); + byte value = (byte) identity.invoke((byte) 0x8); + if (value != (byte) 0x8) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(char.class); + char value = (char) identity.invoke((char) -56); + if (value != (char) -56) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(short.class); + short value = (short) identity.invoke((short) -59); + if (value != (short) -59) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(int.class); + int value = (int) identity.invoke(52); + if (value != 52) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(long.class); + long value = (long) identity.invoke(-76l); + if (value != (long) -76) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(float.class); + float value = (float) identity.invoke(56.0f); + if (value != (float) 56.0f) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(double.class); + double value = (double) identity.invoke((double) 72.0); + if (value != (double) 72.0) { + System.out.println("Unexpected value: " + value); + } + } + + { + MethodHandle identity = MethodHandles.identity(String.class); + String value = (String) identity.invoke("bazman"); + assertEquals("bazman", value); + } + } + + public static void testConstant() throws Throwable { + // int constants. + { + MethodHandle constant = MethodHandles.constant(int.class, 56); + int value = (int) constant.invoke(); + if (value != 56) { + System.out.println("Unexpected value: " + value); + } + + // short constant values are converted to int. + constant = MethodHandles.constant(int.class, (short) 52); + value = (int) constant.invoke(); + if (value != 52) { + System.out.println("Unexpected value: " + value); + } + + // char constant values are converted to int. + constant = MethodHandles.constant(int.class, (char) 'b'); + value = (int) constant.invoke(); + if (value != (int) 'b') { + System.out.println("Unexpected value: " + value); + } + + // int constant values are converted to int. + constant = MethodHandles.constant(int.class, (byte) 0x1); + value = (int) constant.invoke(); + if (value != 1) { + System.out.println("Unexpected value: " + value); + } + + // boolean, float, double and long primitive constants are not convertible + // to int, so the handle creation must fail with a CCE. + try { + MethodHandles.constant(int.class, false); + fail(); + } catch (ClassCastException expected) { + } + + try { + MethodHandles.constant(int.class, 0.1f); + fail(); + } catch (ClassCastException expected) { + } + + try { + MethodHandles.constant(int.class, 0.2); + fail(); + } catch (ClassCastException expected) { + } + + try { + MethodHandles.constant(int.class, 73l); + fail(); + } catch (ClassCastException expected) { + } + } + + // long constants. + { + MethodHandle constant = MethodHandles.constant(long.class, 56l); + long value = (long) constant.invoke(); + if (value != 56l) { + System.out.println("Unexpected value: " + value); + } + + constant = MethodHandles.constant(long.class, (int) 56); + value = (long) constant.invoke(); + if (value != 56l) { + System.out.println("Unexpected value: " + value); + } + } + + // byte constants. + { + MethodHandle constant = MethodHandles.constant(byte.class, (byte) 0x12); + byte value = (byte) constant.invoke(); + if (value != (byte) 0x12) { + System.out.println("Unexpected value: " + value); + } + } + + // boolean constants. + { + MethodHandle constant = MethodHandles.constant(boolean.class, true); + boolean value = (boolean) constant.invoke(); + if (!value) { + System.out.println("Unexpected value: " + value); + } + } + + // char constants. + { + MethodHandle constant = MethodHandles.constant(char.class, 'f'); + char value = (char) constant.invoke(); + if (value != 'f') { + System.out.println("Unexpected value: " + value); + } + } + + // short constants. + { + MethodHandle constant = MethodHandles.constant(short.class, (short) 123); + short value = (short) constant.invoke(); + if (value != (short) 123) { + System.out.println("Unexpected value: " + value); + } + } + + // float constants. + { + MethodHandle constant = MethodHandles.constant(float.class, 56.0f); + float value = (float) constant.invoke(); + if (value != 56.0f) { + System.out.println("Unexpected value: " + value); + } + } + + // double constants. + { + MethodHandle constant = MethodHandles.constant(double.class, 256.0); + double value = (double) constant.invoke(); + if (value != 256.0) { + System.out.println("Unexpected value: " + value); + } + } + + // reference constants. + { + MethodHandle constant = MethodHandles.constant(String.class, "256.0"); + String value = (String) constant.invoke(); + assertEquals("256.0", value); + } } public static void fail() { diff --git a/test/979-invoke-polymorphic-accessors/build b/test/959-invoke-polymorphic-accessors/build index a423ca6b4e..a423ca6b4e 100644 --- a/test/979-invoke-polymorphic-accessors/build +++ b/test/959-invoke-polymorphic-accessors/build diff --git a/test/959-invoke-polymorphic-accessors/expected.txt b/test/959-invoke-polymorphic-accessors/expected.txt new file mode 100644 index 0000000000..de2916b6dd --- /dev/null +++ b/test/959-invoke-polymorphic-accessors/expected.txt @@ -0,0 +1,4 @@ +1515870810 +Passed MethodHandles.Lookup tests for accessors. +Passed MethodHandle.invokeExact() tests for accessors. +Passed MethodHandle.invoke() tests for accessors. diff --git a/test/979-invoke-polymorphic-accessors/info.txt b/test/959-invoke-polymorphic-accessors/info.txt index b2f55f0172..b2f55f0172 100644 --- a/test/979-invoke-polymorphic-accessors/info.txt +++ b/test/959-invoke-polymorphic-accessors/info.txt diff --git a/test/979-invoke-polymorphic-accessors/run b/test/959-invoke-polymorphic-accessors/run index a9f182288c..a9f182288c 100644 --- a/test/979-invoke-polymorphic-accessors/run +++ b/test/959-invoke-polymorphic-accessors/run diff --git a/test/979-invoke-polymorphic-accessors/src/Main.java b/test/959-invoke-polymorphic-accessors/src/Main.java index 6cdcd1006f..824a436f3b 100644 --- a/test/979-invoke-polymorphic-accessors/src/Main.java +++ b/test/959-invoke-polymorphic-accessors/src/Main.java @@ -44,7 +44,26 @@ public class Main { public static final int s_fi = 0x5a5a5a5a; } - public static class InvokeExactTester { + public static class Tester { + public static void assertActualAndExpectedMatch(boolean actual, boolean expected) + throws AssertionError { + if (actual != expected) { + throw new AssertionError("Actual != Expected (" + actual + " != " + expected + ")"); + } + } + + public static void assertTrue(boolean value) throws AssertionError { + if (!value) { + throw new AssertionError("Value is not true"); + } + } + + public static void unreachable() throws Throwable{ + throw new Error("unreachable"); + } + } + + public static class InvokeExactTester extends Tester { private enum PrimitiveType { Boolean, Byte, @@ -64,19 +83,6 @@ public class Main { SGET, } - private static void assertActualAndExpectedMatch(boolean actual, boolean expected) - throws AssertionError { - if (actual != expected) { - throw new AssertionError("Actual != Expected (" + actual + " != " + expected + ")"); - } - } - - private static void assertTrue(boolean value) throws AssertionError { - if (!value) { - throw new AssertionError("Value is not true"); - } - } - static void setByte(MethodHandle m, ValueHolder v, byte value, boolean expectFailure) throws Throwable { boolean exceptionThrown = false; @@ -677,16 +683,26 @@ public class Main { assertTrue(s.equals(ValueHolder.s_l)); } - System.out.println("Passed InvokeExact tests for accessors."); + System.out.println("Passed MethodHandle.invokeExact() tests for accessors."); } } - public static class FindAccessorTester { + public static class FindAccessorTester extends Tester { public static void main() throws Throwable { - ValueHolder valueHolder = new ValueHolder(); + // NB having a static field test here is essential for + // this test. MethodHandles need to ensure the class + // (ValueHolder) is initialized. This happens in the + // invoke-polymorphic dispatch. MethodHandles.Lookup lookup = MethodHandles.lookup(); - - lookup.findStaticGetter(ValueHolder.class, "s_fi", int.class); + try { + MethodHandle mh = lookup.findStaticGetter(ValueHolder.class, "s_fi", int.class); + int initialValue = (int)mh.invokeExact(); + System.out.println(initialValue); + } catch (NoSuchFieldException e) { unreachable(); } + try { + MethodHandle mh = lookup.findStaticSetter(ValueHolder.class, "s_i", int.class); + mh.invokeExact(0); + } catch (NoSuchFieldException e) { unreachable(); } try { lookup.findStaticGetter(ValueHolder.class, "s_fi", byte.class); unreachable(); @@ -713,15 +729,169 @@ public class Main { lookup.findSetter(ValueHolder.class, "m_fi", int.class); unreachable(); } catch (IllegalAccessException e) {} + + System.out.println("Passed MethodHandles.Lookup tests for accessors."); + } + } + + public static class InvokeTester extends Tester { + private static void testStaticGetter() throws Throwable { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + MethodHandle h0 = lookup.findStaticGetter(ValueHolder.class, "s_fi", int.class); + h0.invoke(); + Number t = (Number)h0.invoke(); + int u = (int)h0.invoke(); + Integer v = (Integer)h0.invoke(); + long w = (long)h0.invoke(); + try { + byte x = (byte)h0.invoke(); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + String y = (String)h0.invoke(); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + Long z = (Long)h0.invoke(); + unreachable(); + } catch (WrongMethodTypeException e) {} } - public static void unreachable() throws Throwable{ - throw new Error("unreachable"); + private static void testMemberGetter() throws Throwable { + ValueHolder valueHolder = new ValueHolder(); + MethodHandles.Lookup lookup = MethodHandles.lookup(); + MethodHandle h0 = lookup.findGetter(ValueHolder.class, "m_fi", int.class); + h0.invoke(valueHolder); + Number t = (Number)h0.invoke(valueHolder); + int u = (int)h0.invoke(valueHolder); + Integer v = (Integer)h0.invoke(valueHolder); + long w = (long)h0.invoke(valueHolder); + try { + byte x = (byte)h0.invoke(valueHolder); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + String y = (String)h0.invoke(valueHolder); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + Long z = (Long)h0.invoke(valueHolder); + unreachable(); + } catch (WrongMethodTypeException e) {} + } + + private static void testMemberSetter() throws Throwable { + ValueHolder valueHolder = new ValueHolder(); + MethodHandles.Lookup lookup = MethodHandles.lookup(); + MethodHandle h0 = lookup.findSetter(ValueHolder.class, "m_f", float.class); + h0.invoke(valueHolder, 0.22f); + h0.invoke(valueHolder, new Float(1.11f)); + Number floatNumber = new Float(0.88f); + h0.invoke(valueHolder, floatNumber); + assertTrue(valueHolder.m_f == floatNumber.floatValue()); + + try { + h0.invoke(valueHolder, (Float)null); + unreachable(); + } catch (NullPointerException e) {} + + h0.invoke(valueHolder, (byte)1); + h0.invoke(valueHolder, (short)2); + h0.invoke(valueHolder, 3); + h0.invoke(valueHolder, 4l); + try { + h0.invoke(valueHolder, 0.33); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + Number doubleNumber = new Double(0.89); + h0.invoke(valueHolder, doubleNumber); + unreachable(); + } catch (ClassCastException e) {} + try { + Number doubleNumber = null; + h0.invoke(valueHolder, doubleNumber); + unreachable(); + } catch (NullPointerException e) {} + try { + // Mismatched return type - float != void + float tmp = (float)h0.invoke(valueHolder, 0.45f); + assertTrue(tmp == 0.0); + } catch (Exception e) { unreachable(); } + try { + h0.invoke(valueHolder, "bam"); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + String s = null; + h0.invoke(valueHolder, s); + unreachable(); + } catch (WrongMethodTypeException e) {} + } + + private static void testStaticSetter() throws Throwable { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + MethodHandle h0 = lookup.findStaticSetter(ValueHolder.class, "s_f", float.class); + h0.invoke(0.22f); + h0.invoke(new Float(1.11f)); + Number floatNumber = new Float(0.88f); + h0.invoke(floatNumber); + assertTrue(ValueHolder.s_f == floatNumber.floatValue()); + + try { + h0.invoke((Float)null); + unreachable(); + } catch (NullPointerException e) {} + + h0.invoke((byte)1); + h0.invoke((short)2); + h0.invoke(3); + h0.invoke(4l); + try { + h0.invoke(0.33); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + Number doubleNumber = new Double(0.89); + h0.invoke(doubleNumber); + unreachable(); + } catch (ClassCastException e) {} + try { + Number doubleNumber = new Double(1.01); + doubleNumber = (doubleNumber.doubleValue() != 0.1) ? null : doubleNumber; + h0.invoke(doubleNumber); + unreachable(); + } catch (NullPointerException e) {} + try { + // Mismatched return type - float != void + float tmp = (float)h0.invoke(0.45f); + assertTrue(tmp == 0.0); + } catch (Exception e) { unreachable(); } + try { + h0.invoke("bam"); + unreachable(); + } catch (WrongMethodTypeException e) {} + try { + String s = null; + h0.invoke(s); + unreachable(); + } catch (WrongMethodTypeException e) {} + } + + public static void main() throws Throwable{ + testStaticGetter(); + testMemberGetter(); + testStaticSetter(); + testMemberSetter(); + System.out.println("Passed MethodHandle.invoke() tests for accessors."); } } public static void main(String[] args) throws Throwable { + // FindAccessor test should be the first test class in this + // file to ensure class initialization test is run. FindAccessorTester.main(); InvokeExactTester.main(); + InvokeTester.main(); } } diff --git a/test/979-invoke-polymorphic-accessors/expected.txt b/test/979-invoke-polymorphic-accessors/expected.txt deleted file mode 100644 index 2987b6c5c0..0000000000 --- a/test/979-invoke-polymorphic-accessors/expected.txt +++ /dev/null @@ -1 +0,0 @@ -Passed InvokeExact tests for accessors. diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index ae569f91a3..8f8f99832c 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -384,7 +384,7 @@ TEST_ART_BROKEN_GCSTRESS_RUN_TESTS := \ 908-gc-start-finish \ 913-heaps \ 961-default-iface-resolution-gen \ - 964-default-iface-init-gen \ + 964-default-iface-init-gen ifneq (,$(filter gcstress,$(GC_TYPES))) ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ @@ -608,11 +608,8 @@ TEST_ART_BROKEN_INTERPRETER_READ_BARRIER_RUN_TESTS := # Tests that should fail in the read barrier configuration with the Optimizing compiler (AOT). # 484: Baker's fast path based read barrier compiler instrumentation generates code containing # more parallel moves on x86, thus some Checker assertions may fail. -# 527: On ARM64 and ARM, the read barrier instrumentation does not support the HIntermediateAddress -# instruction yet (b/26601270). TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS := \ - 484-checker-register-hints \ - 527-checker-array-access-split + 484-checker-register-hints # Tests that should fail in the read barrier configuration with JIT (Optimizing compiler). TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS := diff --git a/test/etc/run-test-jar b/test/etc/run-test-jar index cf8db151fe..c525b2b9bf 100755 --- a/test/etc/run-test-jar +++ b/test/etc/run-test-jar @@ -397,12 +397,26 @@ else fi if [ "$HOST" = "n" ]; then - ISA=$(adb shell ls -F /data/dalvik-cache | grep -Ewo "${ARCHITECTURES_PATTERN}") - outcome=$? + # Need to be root to query /data/dalvik-cache + adb root > /dev/null + adb wait-for-device + ISA= + ISA_adb_invocation= + ISA_outcome= + # We iterate a few times to workaround an adb issue. b/32655576 + for i in {1..10}; do + ISA_adb_invocation=$(adb shell ls -F /data/dalvik-cache) + ISA_outcome=$? + ISA=$(echo $ISA_adb_invocation | grep -Ewo "${ARCHITECTURES_PATTERN}") + if [ x"$ISA" != "x" ]; then + break; + fi + done if [ x"$ISA" = "x" ]; then echo "Unable to determine architecture" # Print a few things for helping diagnosing the problem. - echo "adb invocation outcome: $outcome" + echo "adb invocation output: $ISA_adb_invocation" + echo "adb invocation outcome: $ISA_outcome" echo $(adb shell ls -F /data/dalvik-cache) echo $(adb shell ls /data/dalvik-cache) echo ${ARCHITECTURES_PATTERN} diff --git a/test/run-test b/test/run-test index 7a4afafa4e..37eefb34d3 100755 --- a/test/run-test +++ b/test/run-test @@ -758,8 +758,8 @@ fi if [ "$run_checker" = "yes" -a "$target_mode" = "yes" ]; then # We will need to `adb pull` the .cfg output from the target onto the host to # run checker on it. This file can be big. - build_file_size_limit=24576 - run_file_size_limit=24576 + build_file_size_limit=32768 + run_file_size_limit=32768 fi if [ ${USE_JACK} = "false" ]; then # Set ulimit if we build with dx only, Jack can generate big temp files. diff --git a/tools/cpp-define-generator/constant_lockword.def b/tools/cpp-define-generator/constant_lockword.def index 67ed5b5721..08d588505d 100644 --- a/tools/cpp-define-generator/constant_lockword.def +++ b/tools/cpp-define-generator/constant_lockword.def @@ -30,6 +30,10 @@ DEFINE_LOCK_WORD_EXPR(READ_BARRIER_STATE_MASK, uint32_t, kReadBarrierStateMas DEFINE_LOCK_WORD_EXPR(READ_BARRIER_STATE_MASK_TOGGLED, uint32_t, kReadBarrierStateMaskShiftedToggled) DEFINE_LOCK_WORD_EXPR(THIN_LOCK_COUNT_ONE, int32_t, kThinLockCountOne) +DEFINE_LOCK_WORD_EXPR(STATE_FORWARDING_ADDRESS, uint32_t, kStateForwardingAddress) +DEFINE_LOCK_WORD_EXPR(STATE_FORWARDING_ADDRESS_OVERFLOW, uint32_t, kStateForwardingAddressOverflow) +DEFINE_LOCK_WORD_EXPR(STATE_FORWARDING_ADDRESS_SHIFT, uint32_t, kForwardingAddressShift) + DEFINE_LOCK_WORD_EXPR(GC_STATE_MASK_SHIFTED, uint32_t, kGCStateMaskShifted) DEFINE_LOCK_WORD_EXPR(GC_STATE_MASK_SHIFTED_TOGGLED, uint32_t, kGCStateMaskShiftedToggled) DEFINE_LOCK_WORD_EXPR(GC_STATE_SHIFT, int32_t, kGCStateShift) |