diff options
161 files changed, 3059 insertions, 1051 deletions
diff --git a/build/Android.cpplint.mk b/build/Android.cpplint.mk index 2688c049d5..247f4e3470 100644 --- a/build/Android.cpplint.mk +++ b/build/Android.cpplint.mk @@ -18,7 +18,9 @@ include art/build/Android.common_build.mk ART_CPPLINT := $(LOCAL_PATH)/tools/cpplint.py ART_CPPLINT_FILTER := --filter=-whitespace/line_length,-build/include,-readability/function,-readability/streams,-readability/todo,-runtime/references,-runtime/sizeof,-runtime/threadsafe_fn,-runtime/printf -ART_CPPLINT_FLAGS := --root=$(TOP) +# Use `pwd` instead of $TOP for root, $TOP is always . and --root doesn't seem +# to work with a relative path (b/34787652). +ART_CPPLINT_FLAGS := --root=`pwd` ART_CPPLINT_QUIET := --quiet ART_CPPLINT_INGORED := \ runtime/elf.h \ diff --git a/compiler/dex/quick_compiler_callbacks.cc b/compiler/dex/quick_compiler_callbacks.cc index 872f7ea15d..c7e9f4fc07 100644 --- a/compiler/dex/quick_compiler_callbacks.cc +++ b/compiler/dex/quick_compiler_callbacks.cc @@ -16,6 +16,7 @@ #include "quick_compiler_callbacks.h" +#include "driver/compiler_driver.h" #include "verification_results.h" #include "verifier/method_verifier-inl.h" @@ -33,4 +34,17 @@ void QuickCompilerCallbacks::ClassRejected(ClassReference ref) { } } +bool QuickCompilerCallbacks::CanAssumeVerified(ClassReference ref) { + // If we don't have class unloading enabled in the compiler, we will never see class that were + // previously verified. Return false to avoid overhead from the lookup in the compiler driver. + if (!does_class_unloading_) { + return false; + } + DCHECK(compiler_driver_ != nullptr); + // In the case of the quicken filter: avoiding verification of quickened instructions, which the + // verifier doesn't currently support. + // In the case of the verify filter, avoiding verifiying twice. + return compiler_driver_->CanAssumeVerified(ref); +} + } // namespace art diff --git a/compiler/dex/quick_compiler_callbacks.h b/compiler/dex/quick_compiler_callbacks.h index a3a6c0972c..578aff45e5 100644 --- a/compiler/dex/quick_compiler_callbacks.h +++ b/compiler/dex/quick_compiler_callbacks.h @@ -22,6 +22,7 @@ namespace art { +class CompilerDriver; class VerificationResults; class QuickCompilerCallbacks FINAL : public CompilerCallbacks { @@ -53,8 +54,19 @@ class QuickCompilerCallbacks FINAL : public CompilerCallbacks { verification_results_ = verification_results; } + bool CanAssumeVerified(ClassReference ref) OVERRIDE; + + void SetDoesClassUnloading(bool does_class_unloading, CompilerDriver* compiler_driver) + OVERRIDE { + does_class_unloading_ = does_class_unloading; + compiler_driver_ = compiler_driver; + DCHECK(!does_class_unloading || compiler_driver_ != nullptr); + } + private: VerificationResults* verification_results_ = nullptr; + bool does_class_unloading_ = false; + CompilerDriver* compiler_driver_ = nullptr; std::unique_ptr<verifier::VerifierDeps> verifier_deps_; }; diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index bd530ac6a6..037e45840d 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -2879,9 +2879,9 @@ void CompilerDriver::AddCompiledMethod(const MethodReference& method_ref, bool CompilerDriver::GetCompiledClass(ClassReference ref, mirror::Class::Status* status) const { DCHECK(status != nullptr); // The table doesn't know if something wasn't inserted. For this case it will return - // kStatusNotReady. To handle this, just assume anything not verified is not compiled. + // kStatusNotReady. To handle this, just assume anything we didn't try to verify is not compiled. if (!compiled_classes_.Get(DexFileReference(ref.first, ref.second), status) || - *status < mirror::Class::kStatusVerified) { + *status < mirror::Class::kStatusRetryVerificationAtRuntime) { return false; } return true; @@ -3040,4 +3040,10 @@ void CompilerDriver::SetDexFilesForOatFile(const std::vector<const DexFile*>& de } } +bool CompilerDriver::CanAssumeVerified(ClassReference ref) const { + mirror::Class::Status existing = mirror::Class::kStatusNotReady; + compiled_classes_.Get(DexFileReference(ref.first, ref.second), &existing); + return existing >= mirror::Class::kStatusVerified; +} + } // namespace art diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h index d9886a2fba..11808c1be4 100644 --- a/compiler/driver/compiler_driver.h +++ b/compiler/driver/compiler_driver.h @@ -22,6 +22,8 @@ #include <unordered_set> #include <vector> +#include "android-base/strings.h" + #include "arch/instruction_set.h" #include "base/array_ref.h" #include "base/bit_utils.h" @@ -377,6 +379,16 @@ class CompilerDriver { return profile_compilation_info_; } + bool CanAssumeVerified(ClassReference ref) const; + + // Is `boot_image_filename` the name of a core image (small boot + // image used for ART testing only)? + static bool IsCoreImageFilename(const std::string& boot_image_filename) { + // TODO: This is under-approximating... + return android::base::EndsWith(boot_image_filename, "core.art") + || android::base::EndsWith(boot_image_filename, "core-optimizing.art"); + } + private: void PreCompile(jobject class_loader, const std::vector<const DexFile*>& dex_files, diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc index fee6afb91f..392d57c0f2 100644 --- a/compiler/driver/compiler_driver_test.cc +++ b/compiler/driver/compiler_driver_test.cc @@ -23,6 +23,7 @@ #include "art_method-inl.h" #include "class_linker-inl.h" #include "common_compiler_test.h" +#include "compiler_callbacks.h" #include "dex_file.h" #include "dex_file_types.h" #include "gc/heap.h" @@ -366,6 +367,49 @@ TEST_F(CompilerDriverVerifyTest, VerifyCompilation) { CheckVerifiedClass(class_loader, "LSecond;"); } +// Test that a class of status kStatusRetryVerificationAtRuntime is indeed recorded that way in the +// driver. +// Test that checks that classes can be assumed as verified if unloading mode is enabled and +// the class status is at least verified. +TEST_F(CompilerDriverVerifyTest, RetryVerifcationStatusCheckVerified) { + Thread* const self = Thread::Current(); + jobject class_loader; + std::vector<const DexFile*> dex_files; + const DexFile* dex_file = nullptr; + { + ScopedObjectAccess soa(self); + class_loader = LoadDex("ProfileTestMultiDex"); + ASSERT_NE(class_loader, nullptr); + dex_files = GetDexFiles(class_loader); + ASSERT_GT(dex_files.size(), 0u); + dex_file = dex_files.front(); + } + compiler_driver_->SetDexFilesForOatFile(dex_files); + callbacks_->SetDoesClassUnloading(true, compiler_driver_.get()); + ClassReference ref(dex_file, 0u); + // Test that the status is read from the compiler driver as expected. + for (size_t i = mirror::Class::kStatusRetryVerificationAtRuntime; + i < mirror::Class::kStatusMax; + ++i) { + const mirror::Class::Status expected_status = static_cast<mirror::Class::Status>(i); + // Skip unsupported status that are not supposed to be ever recorded. + if (expected_status == mirror::Class::kStatusVerifyingAtRuntime || + expected_status == mirror::Class::kStatusInitializing) { + continue; + } + compiler_driver_->RecordClassStatus(ref, expected_status); + mirror::Class::Status status = {}; + ASSERT_TRUE(compiler_driver_->GetCompiledClass(ref, &status)); + EXPECT_EQ(status, expected_status); + + // Check that we can assume verified if we are a status that is at least verified. + if (status >= mirror::Class::kStatusVerified) { + // Check that the class can be assumed as verified in the compiler driver. + EXPECT_TRUE(callbacks_->CanAssumeVerified(ref)) << status; + } + } +} + // TODO: need check-cast test (when stub complete & we can throw/catch } // namespace art diff --git a/compiler/driver/compiler_options.cc b/compiler/driver/compiler_options.cc index 3cacc2cad7..538845de19 100644 --- a/compiler/driver/compiler_options.cc +++ b/compiler/driver/compiler_options.cc @@ -18,6 +18,8 @@ #include <fstream> +#include "runtime.h" + namespace art { CompilerOptions::CompilerOptions() @@ -30,6 +32,7 @@ CompilerOptions::CompilerOptions() inline_max_code_units_(kUnsetInlineMaxCodeUnits), no_inline_from_(nullptr), boot_image_(false), + core_image_(false), app_image_(false), top_k_profile_threshold_(kDefaultTopKProfileThreshold), debuggable_(false), @@ -55,6 +58,19 @@ CompilerOptions::~CompilerOptions() { // because we don't want to include the PassManagerOptions definition from the header file. } +bool CompilerOptions::EmitRunTimeChecksInDebugMode() const { + // Run-time checks (e.g. Marking Register checks) are only emitted + // in debug mode, and + // - when running on device; or + // - when running on host, but only + // - when compiling the core image (which is used only for testing); or + // - when JIT compiling (only relevant for non-native methods). + // This is to prevent these checks from being emitted into pre-opted + // boot image or apps, as these are compiled with dex2oatd. + return kIsDebugBuild && + (kIsTargetBuild || IsCoreImage() || Runtime::Current()->UseJitCompilation()); +} + void CompilerOptions::ParseHugeMethodMax(const StringPiece& option, UsageFn Usage) { ParseUintOption(option, "--huge-method-max", &huge_method_threshold_, Usage); } diff --git a/compiler/driver/compiler_options.h b/compiler/driver/compiler_options.h index b99263db0e..1e05c4e9aa 100644 --- a/compiler/driver/compiler_options.h +++ b/compiler/driver/compiler_options.h @@ -161,6 +161,9 @@ class CompilerOptions FINAL { return generate_mini_debug_info_; } + // Should run-time checks be emitted in debug mode? + bool EmitRunTimeChecksInDebugMode() const; + bool GetGenerateBuildId() const { return generate_build_id_; } @@ -177,10 +180,19 @@ class CompilerOptions FINAL { return implicit_suspend_checks_; } + // Are we compiling a boot image? bool IsBootImage() const { return boot_image_; } + // Are we compiling a core image (small boot image only used for ART testing)? + bool IsCoreImage() const { + // Ensure that `core_image_` => `boot_image_`. + DCHECK(!core_image_ || boot_image_); + return core_image_; + } + + // Are we compiling an app image? bool IsAppImage() const { return app_image_; } @@ -266,6 +278,7 @@ class CompilerOptions FINAL { const std::vector<const DexFile*>* no_inline_from_; bool boot_image_; + bool core_image_; bool app_image_; // When using a profile file only the top K% of the profiled samples will be compiled. double top_k_profile_threshold_; diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 9e4971ce75..115e722a75 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -1686,6 +1686,10 @@ void ImageWriter::CalculateNewObjectOffsets() { runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs); image_methods_[ImageHeader::kSaveEverythingMethod] = runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything); + image_methods_[ImageHeader::kSaveEverythingMethodForClinit] = + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit); + image_methods_[ImageHeader::kSaveEverythingMethodForSuspendCheck] = + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck); // Visit image methods first to have the main runtime methods in the first image. for (auto* m : image_methods_) { CHECK(m != nullptr); diff --git a/compiler/jni/quick/jni_compiler.cc b/compiler/jni/quick/jni_compiler.cc index b65b93f05f..e7e4647866 100644 --- a/compiler/jni/quick/jni_compiler.cc +++ b/compiler/jni/quick/jni_compiler.cc @@ -219,7 +219,9 @@ static CompiledMethod* ArtJniCompileMethodInternal(CompilerDriver* driver, // Assembler that holds generated instructions std::unique_ptr<JNIMacroAssembler<kPointerSize>> jni_asm = GetMacroAssembler<kPointerSize>(&arena, instruction_set, instruction_set_features); - jni_asm->cfi().SetEnabled(driver->GetCompilerOptions().GenerateAnyDebugInfo()); + const CompilerOptions& compiler_options = driver->GetCompilerOptions(); + jni_asm->cfi().SetEnabled(compiler_options.GenerateAnyDebugInfo()); + jni_asm->SetEmitRunTimeChecksInDebugMode(compiler_options.EmitRunTimeChecksInDebugMode()); // Offsets into data structures // TODO: if cross compiling these offsets are for the host not the target diff --git a/compiler/optimizing/bounds_check_elimination.cc b/compiler/optimizing/bounds_check_elimination.cc index c166deb406..2f96cfa382 100644 --- a/compiler/optimizing/bounds_check_elimination.cc +++ b/compiler/optimizing/bounds_check_elimination.cc @@ -1121,6 +1121,66 @@ class BCEVisitor : public HGraphVisitor { } } + void VisitRem(HRem* instruction) OVERRIDE { + HInstruction* left = instruction->GetLeft(); + HInstruction* right = instruction->GetRight(); + + // Handle 'i % CONST' format expression in array index, e.g: + // array[i % 20]; + if (right->IsIntConstant()) { + int32_t right_const = std::abs(right->AsIntConstant()->GetValue()); + if (right_const == 0) { + return; + } + // The sign of divisor CONST doesn't affect the sign final value range. + // For example: + // if (i > 0) { + // array[i % 10]; // index value range [0, 9] + // array[i % -10]; // index value range [0, 9] + // } + ValueRange* right_range = new (GetGraph()->GetArena()) ValueRange( + GetGraph()->GetArena(), + ValueBound(nullptr, 1 - right_const), + ValueBound(nullptr, right_const - 1)); + + ValueRange* left_range = LookupValueRange(left, left->GetBlock()); + if (left_range != nullptr) { + right_range = left_range->Narrow(right_range); + } + AssignRange(instruction->GetBlock(), instruction, right_range); + return; + } + + // Handle following pattern: + // i0 NullCheck + // i1 ArrayLength[i0] + // i2 DivByZeroCheck [i1] <-- right + // i3 Rem [i5, i2] <-- we are here. + // i4 BoundsCheck [i3,i1] + if (right->IsDivZeroCheck()) { + // if array_length can pass div-by-zero check, + // array_length must be > 0. + right = right->AsDivZeroCheck()->InputAt(0); + } + + // Handle 'i % array.length' format expression in array index, e.g: + // array[(i+7) % array.length]; + if (right->IsArrayLength()) { + ValueBound lower = ValueBound::Min(); // ideally, lower should be '1-array_length'. + ValueBound upper = ValueBound(right, -1); // array_length - 1 + ValueRange* right_range = new (GetGraph()->GetArena()) ValueRange( + GetGraph()->GetArena(), + lower, + upper); + ValueRange* left_range = LookupValueRange(left, left->GetBlock()); + if (left_range != nullptr) { + right_range = left_range->Narrow(right_range); + } + AssignRange(instruction->GetBlock(), instruction, right_range); + return; + } + } + void VisitNewArray(HNewArray* new_array) OVERRIDE { HInstruction* len = new_array->GetLength(); if (!len->IsIntConstant()) { diff --git a/compiler/optimizing/bounds_check_elimination_test.cc b/compiler/optimizing/bounds_check_elimination_test.cc index 575e2fc24a..2aaf05833c 100644 --- a/compiler/optimizing/bounds_check_elimination_test.cc +++ b/compiler/optimizing/bounds_check_elimination_test.cc @@ -951,4 +951,152 @@ TEST_F(BoundsCheckEliminationTest, BubbleSortArrayBoundsElimination) { ASSERT_TRUE(IsRemoved(bounds_check6)); } +// int[] array = new int[10]; +// for (int i=0; i<200; i++) { +// array[i%10] = 10; // Can eliminate +// array[i%1] = 10; // Can eliminate +// array[i%200] = 10; // Cannot eliminate +// array[i%-10] = 10; // Can eliminate +// array[i%array.length] = 10; // Can eliminate +// array[param_i%10] = 10; // Can't eliminate, when param_i < 0 +// } +TEST_F(BoundsCheckEliminationTest, ModArrayBoundsElimination) { + HBasicBlock* entry = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(entry); + graph_->SetEntryBlock(entry); + HInstruction* param_i = new (&allocator_) + HParameterValue(graph_->GetDexFile(), dex::TypeIndex(0), 0, Primitive::kPrimInt); + entry->AddInstruction(param_i); + + HInstruction* constant_0 = graph_->GetIntConstant(0); + HInstruction* constant_1 = graph_->GetIntConstant(1); + HInstruction* constant_10 = graph_->GetIntConstant(10); + HInstruction* constant_200 = graph_->GetIntConstant(200); + HInstruction* constant_minus_10 = graph_->GetIntConstant(-10); + + HBasicBlock* block = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(block); + entry->AddSuccessor(block); + // We pass a bogus constant for the class to avoid mocking one. + HInstruction* new_array = new (&allocator_) HNewArray(constant_10, constant_10, 0); + block->AddInstruction(new_array); + block->AddInstruction(new (&allocator_) HGoto()); + + HBasicBlock* loop_header = new (&allocator_) HBasicBlock(graph_); + HBasicBlock* loop_body = new (&allocator_) HBasicBlock(graph_); + HBasicBlock* exit = new (&allocator_) HBasicBlock(graph_); + + graph_->AddBlock(loop_header); + graph_->AddBlock(loop_body); + graph_->AddBlock(exit); + block->AddSuccessor(loop_header); + loop_header->AddSuccessor(exit); // true successor + loop_header->AddSuccessor(loop_body); // false successor + loop_body->AddSuccessor(loop_header); + + HPhi* phi = new (&allocator_) HPhi(&allocator_, 0, 0, Primitive::kPrimInt); + HInstruction* cmp = new (&allocator_) HGreaterThanOrEqual(phi, constant_200); + HInstruction* if_inst = new (&allocator_) HIf(cmp); + loop_header->AddPhi(phi); + loop_header->AddInstruction(cmp); + loop_header->AddInstruction(if_inst); + phi->AddInput(constant_0); + + ////////////////////////////////////////////////////////////////////////////////// + // LOOP BODY: + // array[i % 10] = 10; + HRem* i_mod_10 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_10, 0); + HBoundsCheck* bounds_check_i_mod_10 = new (&allocator_) HBoundsCheck(i_mod_10, constant_10, 0); + HInstruction* array_set = new (&allocator_) HArraySet( + new_array, bounds_check_i_mod_10, constant_10, Primitive::kPrimInt, 0); + loop_body->AddInstruction(i_mod_10); + loop_body->AddInstruction(bounds_check_i_mod_10); + loop_body->AddInstruction(array_set); + + // array[i % 1] = 10; + HRem* i_mod_1 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_1, 0); + HBoundsCheck* bounds_check_i_mod_1 = new (&allocator_) HBoundsCheck(i_mod_1, constant_10, 0); + array_set = new (&allocator_) HArraySet( + new_array, bounds_check_i_mod_1, constant_10, Primitive::kPrimInt, 0); + loop_body->AddInstruction(i_mod_1); + loop_body->AddInstruction(bounds_check_i_mod_1); + loop_body->AddInstruction(array_set); + + // array[i % 200] = 10; + HRem* i_mod_200 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_1, 0); + HBoundsCheck* bounds_check_i_mod_200 = new (&allocator_) HBoundsCheck(i_mod_200, constant_10, 0); + array_set = new (&allocator_) HArraySet( + new_array, bounds_check_i_mod_200, constant_10, Primitive::kPrimInt, 0); + loop_body->AddInstruction(i_mod_200); + loop_body->AddInstruction(bounds_check_i_mod_200); + loop_body->AddInstruction(array_set); + + // array[i % -10] = 10; + HRem* i_mod_minus_10 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_minus_10, 0); + HBoundsCheck* bounds_check_i_mod_minus_10 = new (&allocator_) HBoundsCheck( + i_mod_minus_10, constant_10, 0); + array_set = new (&allocator_) HArraySet( + new_array, bounds_check_i_mod_minus_10, constant_10, Primitive::kPrimInt, 0); + loop_body->AddInstruction(i_mod_minus_10); + loop_body->AddInstruction(bounds_check_i_mod_minus_10); + loop_body->AddInstruction(array_set); + + // array[i%array.length] = 10; + HNullCheck* null_check = new (&allocator_) HNullCheck(new_array, 0); + HArrayLength* array_length = new (&allocator_) HArrayLength(null_check, 0); + HRem* i_mod_array_length = new (&allocator_) HRem(Primitive::kPrimInt, phi, array_length, 0); + HBoundsCheck* bounds_check_i_mod_array_len = new (&allocator_) HBoundsCheck( + i_mod_array_length, array_length, 0); + array_set = new (&allocator_) HArraySet( + null_check, bounds_check_i_mod_array_len, constant_10, Primitive::kPrimInt, 0); + loop_body->AddInstruction(null_check); + loop_body->AddInstruction(array_length); + loop_body->AddInstruction(i_mod_array_length); + loop_body->AddInstruction(bounds_check_i_mod_array_len); + loop_body->AddInstruction(array_set); + + // array[param_i % 10] = 10; + HRem* param_i_mod_10 = new (&allocator_) HRem(Primitive::kPrimInt, param_i, constant_10, 0); + HBoundsCheck* bounds_check_param_i_mod_10 = new (&allocator_) HBoundsCheck( + param_i_mod_10, constant_10, 0); + array_set = new (&allocator_) HArraySet( + new_array, bounds_check_param_i_mod_10, constant_10, Primitive::kPrimInt, 0); + loop_body->AddInstruction(param_i_mod_10); + loop_body->AddInstruction(bounds_check_param_i_mod_10); + loop_body->AddInstruction(array_set); + + // array[param_i%array.length] = 10; + null_check = new (&allocator_) HNullCheck(new_array, 0); + array_length = new (&allocator_) HArrayLength(null_check, 0); + HRem* param_i_mod_array_length = new (&allocator_) HRem( + Primitive::kPrimInt, param_i, array_length, 0); + HBoundsCheck* bounds_check_param_i_mod_array_len = new (&allocator_) HBoundsCheck( + param_i_mod_array_length, array_length, 0); + array_set = new (&allocator_) HArraySet( + null_check, bounds_check_param_i_mod_array_len, constant_10, Primitive::kPrimInt, 0); + loop_body->AddInstruction(null_check); + loop_body->AddInstruction(array_length); + loop_body->AddInstruction(param_i_mod_array_length); + loop_body->AddInstruction(bounds_check_param_i_mod_array_len); + loop_body->AddInstruction(array_set); + + // i++; + HInstruction* add = new (&allocator_) HAdd(Primitive::kPrimInt, phi, constant_1); + loop_body->AddInstruction(add); + loop_body->AddInstruction(new (&allocator_) HGoto()); + phi->AddInput(add); + ////////////////////////////////////////////////////////////////////////////////// + + exit->AddInstruction(new (&allocator_) HExit()); + + RunBCE(); + + ASSERT_TRUE(IsRemoved(bounds_check_i_mod_10)); + ASSERT_TRUE(IsRemoved(bounds_check_i_mod_1)); + ASSERT_TRUE(IsRemoved(bounds_check_i_mod_200)); + ASSERT_TRUE(IsRemoved(bounds_check_i_mod_minus_10)); + ASSERT_TRUE(IsRemoved(bounds_check_i_mod_array_len)); + ASSERT_FALSE(IsRemoved(bounds_check_param_i_mod_10)); +} + } // namespace art diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 4999950600..3be774a421 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -1595,6 +1595,8 @@ void CodeGeneratorARM64::GenerateFrameEntry() { __ Str(wzr, MemOperand(sp, GetStackOffsetOfShouldDeoptimizeFlag())); } } + + MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void CodeGeneratorARM64::GenerateFrameExit() { @@ -3587,6 +3589,7 @@ void InstructionCodeGeneratorARM64::HandleGoto(HInstruction* got, HBasicBlock* s } if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } if (!codegen_->GoesToNextBlock(block, successor)) { __ B(codegen_->GetLabelOf(successor)); @@ -4391,6 +4394,7 @@ void LocationsBuilderARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { void InstructionCodeGeneratorARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) { @@ -4459,6 +4463,8 @@ void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invok DCHECK(!codegen_->IsLeafMethod()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } + + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -4626,6 +4632,7 @@ void LocationsBuilderARM64::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { void InstructionCodeGeneratorARM64::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { codegen_->GenerateInvokePolymorphicCall(invoke); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeMethodPatch( @@ -4801,27 +4808,37 @@ void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDir DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); if (TryGenerateIntrinsicCode(invoke, codegen_)) { + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); return; } - // Ensure that between the BLR (emitted by GenerateStaticOrDirectCall) and RecordPcInfo there - // are no pools emitted. - EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); - LocationSummary* locations = invoke->GetLocations(); - codegen_->GenerateStaticOrDirectCall( - invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); + { + // Ensure that between the BLR (emitted by GenerateStaticOrDirectCall) and RecordPcInfo there + // are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); + LocationSummary* locations = invoke->GetLocations(); + codegen_->GenerateStaticOrDirectCall( + invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); + } + + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { if (TryGenerateIntrinsicCode(invoke, codegen_)) { + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); return; } - // Ensure that between the BLR (emitted by GenerateVirtualCall) and RecordPcInfo there - // are no pools emitted. - EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); - codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); - DCHECK(!codegen_->IsLeafMethod()); + { + // Ensure that between the BLR (emitted by GenerateVirtualCall) and RecordPcInfo there + // are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); + codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); + DCHECK(!codegen_->IsLeafMethod()); + } + + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( @@ -4895,6 +4912,7 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA HLoadClass::LoadKind load_kind = cls->GetLoadKind(); if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { codegen_->GenerateLoadClassRuntimeCall(cls); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -4995,6 +5013,7 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA } else { __ Bind(slow_path->GetExitLabel()); } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } } @@ -5113,6 +5132,7 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) NO_THREAD codegen_->AddSlowPath(slow_path); __ Cbz(out.X(), slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); return; } case HLoadString::LoadKind::kJitTableAddress: { @@ -5137,6 +5157,7 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) NO_THREAD __ Mov(calling_convention.GetRegisterAt(0).W(), load->GetStringIndex().index_); codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) { @@ -5164,6 +5185,7 @@ void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* ins } else { CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void LocationsBuilderARM64::VisitMul(HMul* mul) { @@ -5260,6 +5282,7 @@ void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) { CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) { @@ -5296,6 +5319,7 @@ void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void LocationsBuilderARM64::VisitNot(HNot* instruction) { @@ -5644,6 +5668,7 @@ void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction return; } GenerateSuspendCheck(instruction, nullptr); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void LocationsBuilderARM64::VisitThrow(HThrow* instruction) { @@ -6021,6 +6046,7 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, @@ -6074,22 +6100,25 @@ void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* ins obj.GetCode()); vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); - EmissionCheckScope guard(GetVIXLAssembler(), - (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); - vixl::aarch64::Label return_address; - __ adr(lr, &return_address); - __ Bind(cbnz_label); - __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. - static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), - "Field LDR must be 1 instruction (4B) before the return address label; " - " 2 instructions (8B) for heap poisoning."); - Register ref_reg = RegisterFrom(ref, Primitive::kPrimNot); - __ ldr(ref_reg, MemOperand(base.X(), offset)); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); + { + EmissionCheckScope guard(GetVIXLAssembler(), + (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); + vixl::aarch64::Label return_address; + __ adr(lr, &return_address); + __ Bind(cbnz_label); + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), + "Field LDR must be 1 instruction (4B) before the return address label; " + " 2 instructions (8B) for heap poisoning."); + Register ref_reg = RegisterFrom(ref, Primitive::kPrimNot); + __ ldr(ref_reg, MemOperand(base.X(), offset)); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); + __ Bind(&return_address); } - GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); - __ Bind(&return_address); + MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__, /* temp_loc */ LocationFrom(ip1)); return; } @@ -6158,19 +6187,22 @@ void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* ins vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); __ Add(temp.X(), obj.X(), Operand(data_offset)); - EmissionCheckScope guard(GetVIXLAssembler(), - (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); - vixl::aarch64::Label return_address; - __ adr(lr, &return_address); - __ Bind(cbnz_label); - __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. - static_assert(BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), - "Array LDR must be 1 instruction (4B) before the return address label; " - " 2 instructions (8B) for heap poisoning."); - __ ldr(ref_reg, MemOperand(temp.X(), index_reg.X(), LSL, scale_factor)); - DCHECK(!needs_null_check); // The thunk cannot handle the null check. - GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); - __ Bind(&return_address); + { + EmissionCheckScope guard(GetVIXLAssembler(), + (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); + vixl::aarch64::Label return_address; + __ adr(lr, &return_address); + __ Bind(cbnz_label); + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + static_assert(BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), + "Array LDR must be 1 instruction (4B) before the return address label; " + " 2 instructions (8B) for heap poisoning."); + __ ldr(ref_reg, MemOperand(temp.X(), index_reg.X(), LSL, scale_factor)); + DCHECK(!needs_null_check); // The thunk cannot handle the null check. + GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); + __ Bind(&return_address); + } + MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__, /* temp_loc */ LocationFrom(ip1)); return; } @@ -6247,6 +6279,7 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* GenerateRawReferenceLoad( instruction, ref, obj, offset, index, scale_factor, needs_null_check, use_load_acquire); __ Bind(slow_path->GetExitLabel()); + MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction, @@ -6303,6 +6336,7 @@ void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* // Fast path: the GC is not marking: nothing to do (the field is // up-to-date, and we don't need to load the reference). __ Bind(slow_path->GetExitLabel()); + MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void CodeGeneratorARM64::GenerateRawReferenceLoad(HInstruction* instruction, @@ -6381,6 +6415,19 @@ void CodeGeneratorARM64::GenerateRawReferenceLoad(HInstruction* instruction, GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); } +void CodeGeneratorARM64::MaybeGenerateMarkingRegisterCheck(int code, Location temp_loc) { + // The following condition is a compile-time one, so it does not have a run-time cost. + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier && kIsDebugBuild) { + // The following condition is a run-time one; it is executed after the + // previous compile-time test, to avoid penalizing non-debug builds. + if (GetCompilerOptions().EmitRunTimeChecksInDebugMode()) { + UseScratchRegisterScope temps(GetVIXLAssembler()); + Register temp = temp_loc.IsValid() ? WRegisterFrom(temp_loc) : temps.AcquireW(); + GetAssembler()->GenerateMarkingRegisterCheck(temp, code); + } + } +} + void CodeGeneratorARM64::GenerateReadBarrierSlow(HInstruction* instruction, Location out, Location ref, diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 584eead81b..c3392097a2 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -687,6 +687,22 @@ class CodeGeneratorARM64 : public CodeGenerator { bool needs_null_check, bool use_load_acquire); + // Emit code checking the status of the Marking Register, and + // aborting the program if MR does not match the value stored in the + // art::Thread object. Code is only emitted in debug mode and if + // CompilerOptions::EmitRunTimeChecksInDebugMode returns true. + // + // Argument `code` is used to identify the different occurrences of + // MaybeGenerateMarkingRegisterCheck in the code generator, and is + // passed to the BRK instruction. + // + // If `temp_loc` is a valid location, it is expected to be a + // register and will be used as a temporary to generate code; + // otherwise, a temporary will be fetched from the core register + // scratch pool. + virtual void MaybeGenerateMarkingRegisterCheck(int code, + Location temp_loc = Location::NoLocation()); + // Generate a read barrier for a heap reference within `instruction` // using a slow path. // diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 430cdde1f7..3d45dd3446 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -94,6 +94,9 @@ constexpr bool kBakerReadBarrierLinkTimeThunksEnableForGcRoots = true; // The reserved entrypoint register for link-time generated thunks. const vixl32::Register kBakerCcEntrypointRegister = r4; +// Using a base helps identify when we hit Marking Register check breakpoints. +constexpr int kMarkingRegisterCheckBreakCodeBaseCode = 0x10; + #ifdef __ #error "ARM Codegen VIXL macro-assembler macro already defined." #endif @@ -2690,6 +2693,8 @@ void CodeGeneratorARMVIXL::GenerateFrameEntry() { __ Mov(temp, 0); GetAssembler()->StoreToOffset(kStoreWord, temp, sp, GetStackOffsetOfShouldDeoptimizeFlag()); } + + MaybeGenerateMarkingRegisterCheck(/* code */ 1); } void CodeGeneratorARMVIXL::GenerateFrameExit() { @@ -2938,6 +2943,7 @@ void InstructionCodeGeneratorARMVIXL::HandleGoto(HInstruction* got, HBasicBlock* } if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 2); } if (!codegen_->GoesToNextBlock(block, successor)) { __ B(codegen_->GetLabelOf(successor)); @@ -3655,6 +3661,7 @@ void LocationsBuilderARMVIXL::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { void InstructionCodeGeneratorARMVIXL::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 3); } void LocationsBuilderARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { @@ -3685,12 +3692,15 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrD DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); if (TryGenerateIntrinsicCode(invoke, codegen_)) { + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 4); return; } LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); + + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 5); } void LocationsBuilderARMVIXL::HandleInvoke(HInvoke* invoke) { @@ -3709,11 +3719,14 @@ void LocationsBuilderARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) { void InstructionCodeGeneratorARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) { if (TryGenerateIntrinsicCode(invoke, codegen_)) { + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 6); return; } codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); + + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 7); } void LocationsBuilderARMVIXL::VisitInvokeInterface(HInvokeInterface* invoke) { @@ -3790,6 +3803,8 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeInterface(HInvokeInterface* inv codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); DCHECK(!codegen_->IsLeafMethod()); } + + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 8); } void LocationsBuilderARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { @@ -3798,6 +3813,7 @@ void LocationsBuilderARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) void InstructionCodeGeneratorARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { codegen_->GenerateInvokePolymorphicCall(invoke); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 9); } void LocationsBuilderARMVIXL::VisitNeg(HNeg* neg) { @@ -5329,6 +5345,7 @@ void InstructionCodeGeneratorARMVIXL::VisitNewInstance(HNewInstance* instruction codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 10); } void LocationsBuilderARMVIXL::VisitNewArray(HNewArray* instruction) { @@ -5348,6 +5365,7 @@ void InstructionCodeGeneratorARMVIXL::VisitNewArray(HNewArray* instruction) { codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); DCHECK(!codegen_->IsLeafMethod()); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 11); } void LocationsBuilderARMVIXL::VisitParameterValue(HParameterValue* instruction) { @@ -6965,6 +6983,7 @@ void InstructionCodeGeneratorARMVIXL::VisitSuspendCheck(HSuspendCheck* instructi return; } GenerateSuspendCheck(instruction, nullptr); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 12); } void InstructionCodeGeneratorARMVIXL::GenerateSuspendCheck(HSuspendCheck* instruction, @@ -7326,6 +7345,7 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ HLoadClass::LoadKind load_kind = cls->GetLoadKind(); if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { codegen_->GenerateLoadClassRuntimeCall(cls); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 13); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -7405,6 +7425,7 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ } else { __ Bind(slow_path->GetExitLabel()); } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 14); } } @@ -7528,6 +7549,7 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THRE codegen_->AddSlowPath(slow_path); __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 15); return; } case HLoadString::LoadKind::kJitTableAddress: { @@ -7548,6 +7570,7 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THRE __ Mov(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 16); } static int32_t GetExceptionTlsOffset() { @@ -8146,6 +8169,7 @@ void InstructionCodeGeneratorARMVIXL::VisitMonitorOperation(HMonitorOperation* i } else { CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 17); } void LocationsBuilderARMVIXL::VisitAnd(HAnd* instruction) { @@ -8647,6 +8671,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } + codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 18); } void CodeGeneratorARMVIXL::MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations) { @@ -8711,31 +8736,34 @@ void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* i base.GetCode(), obj.GetCode(), narrow); vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); - vixl::EmissionCheckScope guard( - GetVIXLAssembler(), - (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); - vixl32::Label return_address; - EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(mr, Operand(0)); - EmitPlaceholderBne(this, bne_label); - ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); - __ ldr(EncodingSize(narrow ? Narrow : Wide), ref_reg, MemOperand(base, offset)); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - // Note: We need a specific width for the unpoisoning NEG. - if (kPoisonHeapReferences) { - if (narrow) { - // The only 16-bit encoding is T1 which sets flags outside IT block (i.e. RSBS, not RSB). - __ rsbs(EncodingSize(Narrow), ref_reg, ref_reg, Operand(0)); - } else { - __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0)); + { + vixl::EmissionCheckScope guard( + GetVIXLAssembler(), + (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); + vixl32::Label return_address; + EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); + __ cmp(mr, Operand(0)); + EmitPlaceholderBne(this, bne_label); + ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); + __ ldr(EncodingSize(narrow ? Narrow : Wide), ref_reg, MemOperand(base, offset)); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); } + // Note: We need a specific width for the unpoisoning NEG. + if (kPoisonHeapReferences) { + if (narrow) { + // The only 16-bit encoding is T1 which sets flags outside IT block (i.e. RSBS, not RSB). + __ rsbs(EncodingSize(Narrow), ref_reg, ref_reg, Operand(0)); + } else { + __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0)); + } + } + __ Bind(&return_address); + DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), + narrow ? BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET + : BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET); } - __ Bind(&return_address); - DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), - narrow ? BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET - : BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET); + MaybeGenerateMarkingRegisterCheck(/* code */ 19, /* temp_loc */ LocationFrom(ip)); return; } @@ -8796,23 +8824,26 @@ void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* i vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); __ Add(data_reg, obj, Operand(data_offset)); - vixl::EmissionCheckScope guard( - GetVIXLAssembler(), - (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); - vixl32::Label return_address; - EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(mr, Operand(0)); - EmitPlaceholderBne(this, bne_label); - ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); - __ ldr(ref_reg, MemOperand(data_reg, index_reg, vixl32::LSL, scale_factor)); - DCHECK(!needs_null_check); // The thunk cannot handle the null check. - // Note: We need a Wide NEG for the unpoisoning. - if (kPoisonHeapReferences) { - __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0)); + { + vixl::EmissionCheckScope guard( + GetVIXLAssembler(), + (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); + vixl32::Label return_address; + EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); + __ cmp(mr, Operand(0)); + EmitPlaceholderBne(this, bne_label); + ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); + __ ldr(ref_reg, MemOperand(data_reg, index_reg, vixl32::LSL, scale_factor)); + DCHECK(!needs_null_check); // The thunk cannot handle the null check. + // Note: We need a Wide NEG for the unpoisoning. + if (kPoisonHeapReferences) { + __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0)); + } + __ Bind(&return_address); + DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), + BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); } - __ Bind(&return_address); - DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), - BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); + MaybeGenerateMarkingRegisterCheck(/* code */ 20, /* temp_loc */ LocationFrom(ip)); return; } @@ -8866,6 +8897,7 @@ void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier(HInstructio // Fast path: the GC is not marking: just load the reference. GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); __ Bind(slow_path->GetExitLabel()); + MaybeGenerateMarkingRegisterCheck(/* code */ 21); } void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction, @@ -8920,6 +8952,7 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction // Fast path: the GC is not marking: nothing to do (the field is // up-to-date, and we don't need to load the reference). __ Bind(slow_path->GetExitLabel()); + MaybeGenerateMarkingRegisterCheck(/* code */ 22); } void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction, @@ -8981,6 +9014,20 @@ void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction, GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); } +void CodeGeneratorARMVIXL::MaybeGenerateMarkingRegisterCheck(int code, Location temp_loc) { + // The following condition is a compile-time one, so it does not have a run-time cost. + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier && kIsDebugBuild) { + // The following condition is a run-time one; it is executed after the + // previous compile-time test, to avoid penalizing non-debug builds. + if (GetCompilerOptions().EmitRunTimeChecksInDebugMode()) { + UseScratchRegisterScope temps(GetVIXLAssembler()); + vixl32::Register temp = temp_loc.IsValid() ? RegisterFrom(temp_loc) : temps.Acquire(); + GetAssembler()->GenerateMarkingRegisterCheck(temp, + kMarkingRegisterCheckBreakCodeBaseCode + code); + } + } +} + void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction, Location out, Location ref, diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index 7ab2993161..5feb33b1e1 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -661,6 +661,28 @@ class CodeGeneratorARMVIXL : public CodeGenerator { ScaleFactor scale_factor, bool needs_null_check); + // Emit code checking the status of the Marking Register, and + // aborting the program if MR does not match the value stored in the + // art::Thread object. Code is only emitted in debug mode and if + // CompilerOptions::EmitRunTimeChecksInDebugMode returns true. + // + // Argument `code` is used to identify the different occurrences of + // MaybeGenerateMarkingRegisterCheck in the code generator, and is + // used together with kMarkingRegisterCheckBreakCodeBaseCode to + // create the value passed to the BKPT instruction. Note that unlike + // in the ARM64 code generator, where `__LINE__` is passed as `code` + // argument to + // CodeGeneratorARM64::MaybeGenerateMarkingRegisterCheck, we cannot + // realistically do that here, as Encoding T1 for the BKPT + // instruction only accepts 8-bit immediate values. + // + // If `temp_loc` is a valid location, it is expected to be a + // register and will be used as a temporary to generate code; + // otherwise, a temporary will be fetched from the core register + // scratch pool. + virtual void MaybeGenerateMarkingRegisterCheck(int code, + Location temp_loc = Location::NoLocation()); + // Generate a read barrier for a heap reference within `instruction` // using a slow path. // diff --git a/compiler/optimizing/code_generator_vector_arm64.cc b/compiler/optimizing/code_generator_vector_arm64.cc index f422b9fc8b..9095ecdf16 100644 --- a/compiler/optimizing/code_generator_vector_arm64.cc +++ b/compiler/optimizing/code_generator_vector_arm64.cc @@ -15,7 +15,9 @@ */ #include "code_generator_arm64.h" + #include "mirror/array-inl.h" +#include "mirror/string.h" using namespace vixl::aarch64; // NOLINT(build/namespaces) diff --git a/compiler/optimizing/code_generator_vector_x86.cc b/compiler/optimizing/code_generator_vector_x86.cc index 14782d70a1..e7aec76aff 100644 --- a/compiler/optimizing/code_generator_vector_x86.cc +++ b/compiler/optimizing/code_generator_vector_x86.cc @@ -15,7 +15,9 @@ */ #include "code_generator_x86.h" + #include "mirror/array-inl.h" +#include "mirror/string.h" namespace art { namespace x86 { diff --git a/compiler/optimizing/code_generator_vector_x86_64.cc b/compiler/optimizing/code_generator_vector_x86_64.cc index 246044ebb8..c7ee81c60d 100644 --- a/compiler/optimizing/code_generator_vector_x86_64.cc +++ b/compiler/optimizing/code_generator_vector_x86_64.cc @@ -15,7 +15,9 @@ */ #include "code_generator_x86_64.h" + #include "mirror/array-inl.h" +#include "mirror/string.h" namespace art { namespace x86_64 { diff --git a/compiler/optimizing/codegen_test_utils.h b/compiler/optimizing/codegen_test_utils.h index cada2e679b..aa4f5da3f0 100644 --- a/compiler/optimizing/codegen_test_utils.h +++ b/compiler/optimizing/codegen_test_utils.h @@ -79,6 +79,21 @@ class CodegenTargetConfig { }; #ifdef ART_ENABLE_CODEGEN_arm +// Special ARM code generator for codegen testing in a limited code +// generation environment (i.e. with no runtime support). +// +// Note: If we want to exercise certains HIR constructions +// (e.g. reference field load in Baker read barrier configuration) in +// codegen tests in the future, we should also: +// - save the Thread Register (R9) and possibly the Marking Register +// (R8) before entering the generated function (both registers are +// callee-save in AAPCS); +// - set these registers to meaningful values before or upon entering +// the generated function (so that generated code using them is +// correct); +// - restore their original values before leaving the generated +// function. + // Provide our own codegen, that ensures the C calling conventions // are preserved. Currently, ART and C do not match as R4 is caller-save // in ART, and callee-save in C. Alternatively, we could use or write @@ -100,6 +115,50 @@ class TestCodeGeneratorARMVIXL : public arm::CodeGeneratorARMVIXL { blocked_core_registers_[arm::R6] = false; blocked_core_registers_[arm::R7] = false; } + + void MaybeGenerateMarkingRegisterCheck(int code ATTRIBUTE_UNUSED, + Location temp_loc ATTRIBUTE_UNUSED) OVERRIDE { + // When turned on, the marking register checks in + // CodeGeneratorARMVIXL::MaybeGenerateMarkingRegisterCheck expects the + // Thread Register and the Marking Register to be set to + // meaningful values. This is not the case in codegen testing, so + // just disable them entirely here (by doing nothing in this + // method). + } +}; +#endif + +#ifdef ART_ENABLE_CODEGEN_arm64 +// Special ARM64 code generator for codegen testing in a limited code +// generation environment (i.e. with no runtime support). +// +// Note: If we want to exercise certains HIR constructions +// (e.g. reference field load in Baker read barrier configuration) in +// codegen tests in the future, we should also: +// - save the Thread Register (X19) and possibly the Marking Register +// (X20) before entering the generated function (both registers are +// callee-save in AAPCS64); +// - set these registers to meaningful values before or upon entering +// the generated function (so that generated code using them is +// correct); +// - restore their original values before leaving the generated +// function. +class TestCodeGeneratorARM64 : public arm64::CodeGeneratorARM64 { + public: + TestCodeGeneratorARM64(HGraph* graph, + const Arm64InstructionSetFeatures& isa_features, + const CompilerOptions& compiler_options) + : arm64::CodeGeneratorARM64(graph, isa_features, compiler_options) {} + + void MaybeGenerateMarkingRegisterCheck(int codem ATTRIBUTE_UNUSED, + Location temp_loc ATTRIBUTE_UNUSED) OVERRIDE { + // When turned on, the marking register checks in + // CodeGeneratorARM64::MaybeGenerateMarkingRegisterCheck expect the + // Thread Register and the Marking Register to be set to + // meaningful values. This is not the case in codegen testing, so + // just disable them entirely here (by doing nothing in this + // method). + } }; #endif @@ -263,7 +322,8 @@ static void RunCode(CodegenTargetConfig target_config, bool has_result, Expected expected) { CompilerOptions compiler_options; - std::unique_ptr<CodeGenerator> codegen(target_config.CreateCodeGenerator(graph, compiler_options)); + std::unique_ptr<CodeGenerator> codegen(target_config.CreateCodeGenerator(graph, + compiler_options)); RunCode(codegen.get(), graph, hook_before_codegen, has_result, expected); } @@ -280,9 +340,8 @@ CodeGenerator* create_codegen_arm_vixl32(HGraph* graph, const CompilerOptions& c CodeGenerator* create_codegen_arm64(HGraph* graph, const CompilerOptions& compiler_options) { std::unique_ptr<const Arm64InstructionSetFeatures> features_arm64( Arm64InstructionSetFeatures::FromCppDefines()); - return new (graph->GetArena()) arm64::CodeGeneratorARM64(graph, - *features_arm64.get(), - compiler_options); + return new (graph->GetArena()) + TestCodeGeneratorARM64(graph, *features_arm64.get(), compiler_options); } #endif diff --git a/compiler/optimizing/induction_var_range.h b/compiler/optimizing/induction_var_range.h index ab1772bf15..0b980f596a 100644 --- a/compiler/optimizing/induction_var_range.h +++ b/compiler/optimizing/induction_var_range.h @@ -151,6 +151,16 @@ class InductionVarRange { } /** + * Checks if the given phi instruction has been classified as anything by + * induction variable analysis. Returns false for anything that cannot be + * classified statically, such as reductions or other complex cycles. + */ + bool IsClassified(HPhi* phi) const { + HLoopInformation* lp = phi->GetBlock()->GetLoopInformation(); // closest enveloping loop + return (lp != nullptr) && (induction_analysis_->LookupInfo(lp, phi) != nullptr); + } + + /** * Checks if header logic of a loop terminates. Sets trip-count tc if known. */ bool IsFinite(HLoopInformation* loop, /*out*/ int64_t* tc) const; diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index 4cea6dfdfb..2669d97d82 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -22,6 +22,7 @@ #include "entrypoints/quick/quick_entrypoints.h" #include "intrinsics.h" #include "mirror/array-inl.h" +#include "mirror/object_array-inl.h" #include "mirror/string.h" #include "scoped_thread_state_change-inl.h" #include "thread.h" diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index d785567e0f..74be954a75 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -22,6 +22,7 @@ #include "entrypoints/quick/quick_entrypoints.h" #include "intrinsics.h" #include "mirror/array-inl.h" +#include "mirror/object_array-inl.h" #include "mirror/string.h" #include "scoped_thread_state_change-inl.h" #include "thread.h" diff --git a/compiler/optimizing/load_store_analysis.h b/compiler/optimizing/load_store_analysis.h index 86fb8e0165..a2c17944f2 100644 --- a/compiler/optimizing/load_store_analysis.h +++ b/compiler/optimizing/load_store_analysis.h @@ -466,6 +466,11 @@ class HeapLocationCollector : public HGraphVisitor { CreateReferenceInfoForReferenceType(new_instance); } + void VisitNewArray(HNewArray* new_array) OVERRIDE { + // Any references appearing in the ref_info_array_ so far cannot alias with new_array. + CreateReferenceInfoForReferenceType(new_array); + } + void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* instruction) OVERRIDE { CreateReferenceInfoForReferenceType(instruction); } @@ -478,6 +483,22 @@ class HeapLocationCollector : public HGraphVisitor { CreateReferenceInfoForReferenceType(instruction); } + void VisitInvokeUnresolved(HInvokeUnresolved* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + + void VisitInvokePolymorphic(HInvokePolymorphic* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + + void VisitLoadString(HLoadString* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + + void VisitPhi(HPhi* instruction) OVERRIDE { + CreateReferenceInfoForReferenceType(instruction); + } + void VisitParameterValue(HParameterValue* instruction) OVERRIDE { CreateReferenceInfoForReferenceType(instruction); } diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index 0ef7dcdb59..027ba7741c 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -256,6 +256,35 @@ static bool IsAddConst(HInstruction* instruction, return false; } +// Detect reductions of the following forms, +// under assumption phi has only *one* use: +// x = x_phi + .. +// x = x_phi - .. +// x = max(x_phi, ..) +// x = min(x_phi, ..) +static bool HasReductionFormat(HInstruction* reduction, HInstruction* phi) { + if (reduction->IsAdd()) { + return reduction->InputAt(0) == phi || reduction->InputAt(1) == phi; + } else if (reduction->IsSub()) { + return reduction->InputAt(0) == phi; + } else if (reduction->IsInvokeStaticOrDirect()) { + switch (reduction->AsInvokeStaticOrDirect()->GetIntrinsic()) { + case Intrinsics::kMathMinIntInt: + case Intrinsics::kMathMinLongLong: + case Intrinsics::kMathMinFloatFloat: + case Intrinsics::kMathMinDoubleDouble: + case Intrinsics::kMathMaxIntInt: + case Intrinsics::kMathMaxLongLong: + case Intrinsics::kMathMaxFloatFloat: + case Intrinsics::kMathMaxDoubleDouble: + return reduction->InputAt(0) == phi || reduction->InputAt(1) == phi; + default: + return false; + } + } + return false; +} + // Test vector restrictions. static bool HasVectorRestrictions(uint64_t restrictions, uint64_t tested) { return (restrictions & tested) != 0; @@ -280,12 +309,11 @@ static bool CheckInductionSetFullyRemoved(ArenaSet<HInstruction*>* iset) { return false; } } - return true; } // -// Class methods. +// Public methods. // HLoopOptimization::HLoopOptimization(HGraph* graph, @@ -299,7 +327,7 @@ HLoopOptimization::HLoopOptimization(HGraph* graph, top_loop_(nullptr), last_loop_(nullptr), iset_(nullptr), - induction_simplication_count_(0), + reductions_(nullptr), simplified_(false), vector_length_(0), vector_refs_(nullptr), @@ -333,6 +361,10 @@ void HLoopOptimization::Run() { last_loop_ = top_loop_ = nullptr; } +// +// Loop setup and traversal. +// + void HLoopOptimization::LocalRun() { // Build the linear order using the phase-local allocator. This step enables building // a loop hierarchy that properly reflects the outer-inner and previous-next relation. @@ -351,17 +383,21 @@ void HLoopOptimization::LocalRun() { // should use the global allocator. if (top_loop_ != nullptr) { ArenaSet<HInstruction*> iset(loop_allocator_->Adapter(kArenaAllocLoopOptimization)); + ArenaSafeMap<HInstruction*, HInstruction*> reds( + std::less<HInstruction*>(), loop_allocator_->Adapter(kArenaAllocLoopOptimization)); ArenaSet<ArrayReference> refs(loop_allocator_->Adapter(kArenaAllocLoopOptimization)); ArenaSafeMap<HInstruction*, HInstruction*> map( std::less<HInstruction*>(), loop_allocator_->Adapter(kArenaAllocLoopOptimization)); // Attach. iset_ = &iset; + reductions_ = &reds; vector_refs_ = &refs; vector_map_ = ↦ // Traverse. TraverseLoopsInnerToOuter(top_loop_); // Detach. iset_ = nullptr; + reductions_ = nullptr; vector_refs_ = nullptr; vector_map_ = nullptr; } @@ -414,16 +450,12 @@ void HLoopOptimization::RemoveLoop(LoopNode* node) { } } -void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { +bool HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { + bool changed = false; for ( ; node != nullptr; node = node->next) { - // Visit inner loops first. - uint32_t current_induction_simplification_count = induction_simplication_count_; - if (node->inner != nullptr) { - TraverseLoopsInnerToOuter(node->inner); - } - // Recompute induction information of this loop if the induction - // of any inner loop has been simplified. - if (current_induction_simplification_count != induction_simplication_count_) { + // Visit inner loops first. Recompute induction information for this + // loop if the induction of any inner loop has changed. + if (TraverseLoopsInnerToOuter(node->inner)) { induction_range_.ReVisit(node->loop_info); } // Repeat simplifications in the loop-body until no more changes occur. @@ -433,12 +465,14 @@ void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { simplified_ = false; SimplifyInduction(node); SimplifyBlocks(node); + changed = simplified_ || changed; } while (simplified_); // Optimize inner loop. if (node->inner == nullptr) { - OptimizeInnerLoop(node); + changed = OptimizeInnerLoop(node) || changed; } } + return changed; } // @@ -455,20 +489,18 @@ void HLoopOptimization::SimplifyInduction(LoopNode* node) { // for (int i = 0; i < 10; i++, k++) { .... no k .... } return k; for (HInstructionIterator it(header->GetPhis()); !it.Done(); it.Advance()) { HPhi* phi = it.Current()->AsPhi(); - iset_->clear(); // prepare phi induction if (TrySetPhiInduction(phi, /*restrict_uses*/ true) && TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ false)) { // Note that it's ok to have replaced uses after the loop with the last value, without // being able to remove the cycle. Environment uses (which are the reason we may not be - // able to remove the cycle) within the loop will still hold the right value. + // able to remove the cycle) within the loop will still hold the right value. We must + // have tried first, however, to replace outside uses. if (CanRemoveCycle()) { + simplified_ = true; for (HInstruction* i : *iset_) { RemoveFromCycle(i); } - - // Check that there are no records of the deleted instructions. DCHECK(CheckInductionSetFullyRemoved(iset_)); - simplified_ = true; } } } @@ -511,21 +543,20 @@ void HLoopOptimization::SimplifyBlocks(LoopNode* node) { } } -void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { +bool HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { HBasicBlock* header = node->loop_info->GetHeader(); HBasicBlock* preheader = node->loop_info->GetPreHeader(); // Ensure loop header logic is finite. int64_t trip_count = 0; if (!induction_range_.IsFinite(node->loop_info, &trip_count)) { - return; + return false; } - // Ensure there is only a single loop-body (besides the header). HBasicBlock* body = nullptr; for (HBlocksInLoopIterator it(*node->loop_info); !it.Done(); it.Advance()) { if (it.Current() != header) { if (body != nullptr) { - return; + return false; } body = it.Current(); } @@ -533,27 +564,27 @@ void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { CHECK(body != nullptr); // Ensure there is only a single exit point. if (header->GetSuccessors().size() != 2) { - return; + return false; } HBasicBlock* exit = (header->GetSuccessors()[0] == body) ? header->GetSuccessors()[1] : header->GetSuccessors()[0]; // Ensure exit can only be reached by exiting loop. if (exit->GetPredecessors().size() != 1) { - return; + return false; } // Detect either an empty loop (no side effects other than plain iteration) or // a trivial loop (just iterating once). Replace subsequent index uses, if any, // with the last value and remove the loop, possibly after unrolling its body. - HInstruction* phi = header->GetFirstPhi(); - iset_->clear(); // prepare phi induction - if (TrySetSimpleLoopHeader(header)) { + HPhi* main_phi = nullptr; + if (TrySetSimpleLoopHeader(header, &main_phi)) { bool is_empty = IsEmptyBody(body); - if ((is_empty || trip_count == 1) && - TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ true)) { + if (reductions_->empty() && // TODO: possible with some effort + (is_empty || trip_count == 1) && + TryAssignLastValue(node->loop_info, main_phi, preheader, /*collect_loop_uses*/ true)) { if (!is_empty) { // Unroll the loop-body, which sees initial value of the index. - phi->ReplaceWith(phi->InputAt(0)); + main_phi->ReplaceWith(main_phi->InputAt(0)); preheader->MergeInstructionsWith(body); } body->DisconnectAndDelete(); @@ -566,21 +597,20 @@ void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { preheader->AddDominatedBlock(exit); exit->SetDominator(preheader); RemoveLoop(node); // update hierarchy - return; + return true; } } - // Vectorize loop, if possible and valid. - if (kEnableVectorization) { - iset_->clear(); // prepare phi induction - if (TrySetSimpleLoopHeader(header) && - ShouldVectorize(node, body, trip_count) && - TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ true)) { - Vectorize(node, body, exit, trip_count); - graph_->SetHasSIMD(true); // flag SIMD usage - return; - } + if (kEnableVectorization && + TrySetSimpleLoopHeader(header, &main_phi) && + reductions_->empty() && // TODO: possible with some effort + ShouldVectorize(node, body, trip_count) && + TryAssignLastValue(node->loop_info, main_phi, preheader, /*collect_loop_uses*/ true)) { + Vectorize(node, body, exit, trip_count); + graph_->SetHasSIMD(true); // flag SIMD usage + return true; } + return false; } // @@ -621,6 +651,8 @@ bool HLoopOptimization::ShouldVectorize(LoopNode* node, HBasicBlock* block, int6 // aliased, as well as the property that references either point to the same // array or to two completely disjoint arrays, i.e., no partial aliasing. // Other than a few simply heuristics, no detailed subscript analysis is done. + // The scan over references also finds a suitable dynamic loop peeling candidate. + const ArrayReference* candidate = nullptr; for (auto i = vector_refs_->begin(); i != vector_refs_->end(); ++i) { for (auto j = i; ++j != vector_refs_->end(); ) { if (i->type == j->type && (i->lhs || j->lhs)) { @@ -656,7 +688,7 @@ bool HLoopOptimization::ShouldVectorize(LoopNode* node, HBasicBlock* block, int6 } // Consider dynamic loop peeling for alignment. - SetPeelingCandidate(trip_count); + SetPeelingCandidate(candidate, trip_count); // Success! return true; @@ -679,14 +711,15 @@ void HLoopOptimization::Vectorize(LoopNode* node, bool needs_cleanup = trip_count == 0 || (trip_count % chunk) != 0; // Adjust vector bookkeeping. - iset_->clear(); // prepare phi induction - bool is_simple_loop_header = TrySetSimpleLoopHeader(header); // fills iset_ + HPhi* main_phi = nullptr; + bool is_simple_loop_header = TrySetSimpleLoopHeader(header, &main_phi); // refills sets DCHECK(is_simple_loop_header); vector_header_ = header; vector_body_ = block; - // Generate dynamic loop peeling trip count, if needed: - // ptc = <peeling-needed-for-candidate> + // Generate dynamic loop peeling trip count, if needed, under the assumption + // that the Android runtime guarantees at least "component size" alignment: + // ptc = (ALIGN - (&a[initial] % ALIGN)) / type-size HInstruction* ptc = nullptr; if (vector_peeling_candidate_ != nullptr) { DCHECK_LT(vector_length_, trip_count) << "dynamic peeling currently requires known trip count"; @@ -775,6 +808,7 @@ void HLoopOptimization::Vectorize(LoopNode* node, while (!header->GetFirstInstruction()->IsGoto()) { header->RemoveInstruction(header->GetFirstInstruction()); } + // Update loop hierarchy: the old header now resides in the same outer loop // as the old preheader. Note that we don't bother putting sequential // loops back in the hierarchy at this point. @@ -841,13 +875,12 @@ void HLoopOptimization::GenerateNewLoop(LoopNode* node, vector_index_ = new (global_allocator_) HAdd(induc_type, vector_index_, step); Insert(vector_body_, vector_index_); } - // Finalize phi for the loop index. + // Finalize phi inputs for the loop index. phi->AddInput(lo); phi->AddInput(vector_index_); vector_index_ = phi; } -// TODO: accept reductions at left-hand-side, mixed-type store idioms, etc. bool HLoopOptimization::VectorizeDef(LoopNode* node, HInstruction* instruction, bool generate_code) { @@ -1118,7 +1151,7 @@ bool HLoopOptimization::TrySetVectorType(Primitive::Type type, uint64_t* restric case kArm: case kThumb2: // Allow vectorization for all ARM devices, because Android assumes that - // ARM 32-bit always supports advanced SIMD. + // ARM 32-bit always supports advanced SIMD (64-bit SIMD). switch (type) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: @@ -1137,7 +1170,7 @@ bool HLoopOptimization::TrySetVectorType(Primitive::Type type, uint64_t* restric return false; case kArm64: // Allow vectorization for all ARM devices, because Android assumes that - // ARMv8 AArch64 always supports advanced SIMD. + // ARMv8 AArch64 always supports advanced SIMD (128-bit SIMD). switch (type) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: @@ -1162,7 +1195,7 @@ bool HLoopOptimization::TrySetVectorType(Primitive::Type type, uint64_t* restric } case kX86: case kX86_64: - // Allow vectorization for SSE4-enabled X86 devices only (128-bit vectors). + // Allow vectorization for SSE4.1-enabled X86 devices only (128-bit SIMD). if (features->AsX86InstructionSetFeatures()->HasSSE4_1()) { switch (type) { case Primitive::kPrimBoolean: @@ -1310,8 +1343,6 @@ void HLoopOptimization::GenerateVecMem(HInstruction* org, global_allocator_, base, opa, type, vector_length_, is_string_char_at); } // Known dynamically enforced alignment? - // TODO: detect offset + constant differences. - // TODO: long run, static alignment analysis? if (vector_peeling_candidate_ != nullptr && vector_peeling_candidate_->base == base && vector_peeling_candidate_->offset == offset) { @@ -1586,9 +1617,11 @@ bool HLoopOptimization::IsVectorizationProfitable(int64_t trip_count) { return true; } -void HLoopOptimization::SetPeelingCandidate(int64_t trip_count ATTRIBUTE_UNUSED) { +void HLoopOptimization::SetPeelingCandidate(const ArrayReference* candidate, + int64_t trip_count ATTRIBUTE_UNUSED) { // Current heuristic: none. // TODO: implement + vector_peeling_candidate_ = candidate; } uint32_t HLoopOptimization::GetUnrollingFactor(HBasicBlock* block, int64_t trip_count) { @@ -1616,13 +1649,17 @@ uint32_t HLoopOptimization::GetUnrollingFactor(HBasicBlock* block, int64_t trip_ // bool HLoopOptimization::TrySetPhiInduction(HPhi* phi, bool restrict_uses) { + // Start with empty phi induction. + iset_->clear(); + // Special case Phis that have equivalent in a debuggable setup. Our graph checker isn't // smart enough to follow strongly connected components (and it's probably not worth // it to make it so). See b/33775412. if (graph_->IsDebuggable() && phi->HasEquivalentPhi()) { return false; } - DCHECK(iset_->empty()); + + // Lookup phi induction cycle. ArenaSet<HInstruction*>* set = induction_range_.LookupCycle(phi); if (set != nullptr) { for (HInstruction* i : *set) { @@ -1634,6 +1671,7 @@ bool HLoopOptimization::TrySetPhiInduction(HPhi* phi, bool restrict_uses) { } else if (!i->IsRemovable()) { return false; } else if (i != phi && restrict_uses) { + // Deal with regular uses. for (const HUseListNode<HInstruction*>& use : i->GetUses()) { if (set->find(use.GetUser()) == set->end()) { return false; @@ -1647,17 +1685,65 @@ bool HLoopOptimization::TrySetPhiInduction(HPhi* phi, bool restrict_uses) { return false; } -// Find: phi: Phi(init, addsub) -// s: SuspendCheck -// c: Condition(phi, bound) -// i: If(c) -// TODO: Find a less pattern matching approach? -bool HLoopOptimization::TrySetSimpleLoopHeader(HBasicBlock* block) { +bool HLoopOptimization::TrySetPhiReduction(HPhi* phi) { DCHECK(iset_->empty()); - HInstruction* phi = block->GetFirstPhi(); - if (phi != nullptr && - phi->GetNext() == nullptr && - TrySetPhiInduction(phi->AsPhi(), /*restrict_uses*/ false)) { + // Only unclassified phi cycles are candidates for reductions. + if (induction_range_.IsClassified(phi)) { + return false; + } + // Accept operations like x = x + .., provided that the phi and the reduction are + // used exactly once inside the loop, and by each other. + HInputsRef inputs = phi->GetInputs(); + if (inputs.size() == 2) { + HInstruction* reduction = inputs[1]; + if (HasReductionFormat(reduction, phi)) { + HLoopInformation* loop_info = phi->GetBlock()->GetLoopInformation(); + int32_t use_count = 0; + bool single_use_inside_loop = + // Reduction update only used by phi. + reduction->GetUses().HasExactlyOneElement() && + !reduction->HasEnvironmentUses() && + // Reduction update is only use of phi inside the loop. + IsOnlyUsedAfterLoop(loop_info, phi, /*collect_loop_uses*/ true, &use_count) && + iset_->size() == 1; + iset_->clear(); // leave the way you found it + if (single_use_inside_loop) { + // Link reduction back, and start recording feed value. + reductions_->Put(reduction, phi); + reductions_->Put(phi, phi->InputAt(0)); + return true; + } + } + } + return false; +} + +bool HLoopOptimization::TrySetSimpleLoopHeader(HBasicBlock* block, /*out*/ HPhi** main_phi) { + // Start with empty phi induction and reductions. + iset_->clear(); + reductions_->clear(); + + // Scan the phis to find the following (the induction structure has already + // been optimized, so we don't need to worry about trivial cases): + // (1) optional reductions in loop, + // (2) the main induction, used in loop control. + HPhi* phi = nullptr; + for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { + if (TrySetPhiReduction(it.Current()->AsPhi())) { + continue; + } else if (phi == nullptr) { + // Found the first candidate for main induction. + phi = it.Current()->AsPhi(); + } else { + return false; + } + } + + // Then test for a typical loopheader: + // s: SuspendCheck + // c: Condition(phi, bound) + // i: If(c) + if (phi != nullptr && TrySetPhiInduction(phi, /*restrict_uses*/ false)) { HInstruction* s = block->GetFirstInstruction(); if (s != nullptr && s->IsSuspendCheck()) { HInstruction* c = s->GetNext(); @@ -1669,6 +1755,7 @@ bool HLoopOptimization::TrySetSimpleLoopHeader(HBasicBlock* block) { if (i != nullptr && i->IsIf() && i->InputAt(0) == c) { iset_->insert(c); iset_->insert(s); + *main_phi = phi; return true; } } @@ -1692,6 +1779,7 @@ bool HLoopOptimization::IsEmptyBody(HBasicBlock* block) { bool HLoopOptimization::IsUsedOutsideLoop(HLoopInformation* loop_info, HInstruction* instruction) { + // Deal with regular uses. for (const HUseListNode<HInstruction*>& use : instruction->GetUses()) { if (use.GetUser()->GetBlock()->GetLoopInformation() != loop_info) { return true; @@ -1704,6 +1792,7 @@ bool HLoopOptimization::IsOnlyUsedAfterLoop(HLoopInformation* loop_info, HInstruction* instruction, bool collect_loop_uses, /*out*/ int32_t* use_count) { + // Deal with regular uses. for (const HUseListNode<HInstruction*>& use : instruction->GetUses()) { HInstruction* user = use.GetUser(); if (iset_->find(user) == iset_->end()) { // not excluded? @@ -1729,6 +1818,7 @@ bool HLoopOptimization::TryReplaceWithLastValue(HLoopInformation* loop_info, // Try to replace outside uses with the last value. if (induction_range_.CanGenerateLastValue(instruction)) { HInstruction* replacement = induction_range_.GenerateLastValue(instruction, graph_, block); + // Deal with regular uses. const HUseList<HInstruction*>& uses = instruction->GetUses(); for (auto it = uses.begin(), end = uses.end(); it != end;) { HInstruction* user = it->GetUser(); @@ -1744,6 +1834,7 @@ bool HLoopOptimization::TryReplaceWithLastValue(HLoopInformation* loop_info, induction_range_.Replace(user, instruction, replacement); // update induction } } + // Deal with environment uses. const HUseList<HEnvironment*>& env_uses = instruction->GetEnvUses(); for (auto it = env_uses.begin(), end = env_uses.end(); it != end;) { HEnvironment* user = it->GetUser(); @@ -1759,7 +1850,6 @@ bool HLoopOptimization::TryReplaceWithLastValue(HLoopInformation* loop_info, } } } - induction_simplication_count_++; return true; } return false; diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h index de4bd85fc8..49be8a3fb4 100644 --- a/compiler/optimizing/loop_optimization.h +++ b/compiler/optimizing/loop_optimization.h @@ -104,18 +104,33 @@ class HLoopOptimization : public HOptimization { bool lhs; // def/use }; + // // Loop setup and traversal. + // + void LocalRun(); void AddLoop(HLoopInformation* loop_info); void RemoveLoop(LoopNode* node); - void TraverseLoopsInnerToOuter(LoopNode* node); + // Traverses all loops inner to outer to perform simplifications and optimizations. + // Returns true if loops nested inside current loop (node) have changed. + bool TraverseLoopsInnerToOuter(LoopNode* node); + + // // Optimization. + // + void SimplifyInduction(LoopNode* node); void SimplifyBlocks(LoopNode* node); - void OptimizeInnerLoop(LoopNode* node); + // Performs optimizations specific to inner loop (empty loop removal, + // unrolling, vectorization). Returns true if anything changed. + bool OptimizeInnerLoop(LoopNode* node); + + // // Vectorization analysis and synthesis. + // + bool ShouldVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count); void Vectorize(LoopNode* node, HBasicBlock* block, HBasicBlock* exit, int64_t trip_count); void GenerateNewLoop(LoopNode* node, @@ -155,12 +170,20 @@ class HLoopOptimization : public HOptimization { // Vectorization heuristics. bool IsVectorizationProfitable(int64_t trip_count); - void SetPeelingCandidate(int64_t trip_count); + void SetPeelingCandidate(const ArrayReference* candidate, int64_t trip_count); uint32_t GetUnrollingFactor(HBasicBlock* block, int64_t trip_count); + // // Helpers. + // + bool TrySetPhiInduction(HPhi* phi, bool restrict_uses); - bool TrySetSimpleLoopHeader(HBasicBlock* block); + bool TrySetPhiReduction(HPhi* phi); + + // Detects loop header with a single induction (returned in main_phi), possibly + // other phis for reductions, but no other side effects. Returns true on success. + bool TrySetSimpleLoopHeader(HBasicBlock* block, /*out*/ HPhi** main_phi); + bool IsEmptyBody(HBasicBlock* block); bool IsOnlyUsedAfterLoop(HLoopInformation* loop_info, HInstruction* instruction, @@ -200,10 +223,12 @@ class HLoopOptimization : public HOptimization { // Contents reside in phase-local heap memory. ArenaSet<HInstruction*>* iset_; - // Counter that tracks how many induction cycles have been simplified. Useful - // to trigger incremental updates of induction variable analysis of outer loops - // when the induction of inner loops has changed. - uint32_t induction_simplication_count_; + // Temporary bookkeeping of reduction instructions. Mapping is two-fold: + // (1) reductions in the loop-body are mapped back to their phi definition, + // (2) phi definitions are mapped to their initial value (updated during + // code generation to feed the proper values into the new chain). + // Contents reside in phase-local heap memory. + ArenaSafeMap<HInstruction*, HInstruction*>* reductions_; // Flag that tracks if any simplifications have occurred. bool simplified_; diff --git a/compiler/optimizing/loop_optimization_test.cc b/compiler/optimizing/loop_optimization_test.cc index 5b9350689e..b5b03d8f26 100644 --- a/compiler/optimizing/loop_optimization_test.cc +++ b/compiler/optimizing/loop_optimization_test.cc @@ -195,4 +195,44 @@ TEST_F(LoopOptimizationTest, LoopNestWithSequence) { EXPECT_EQ("[[[[[[[[[[][][][][][][][][][]]]]]]]]]]", LoopStructure()); } +// Check that SimplifyLoop() doesn't invalidate data flow when ordering loop headers' +// predecessors. +TEST_F(LoopOptimizationTest, SimplifyLoop) { + // Can't use AddLoop as we want special order for blocks predecessors. + HBasicBlock* header = new (&allocator_) HBasicBlock(graph_); + HBasicBlock* body = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(header); + graph_->AddBlock(body); + + // Control flow: make a loop back edge first in the list of predecessors. + entry_block_->RemoveSuccessor(return_block_); + body->AddSuccessor(header); + entry_block_->AddSuccessor(header); + header->AddSuccessor(body); + header->AddSuccessor(return_block_); + DCHECK(header->GetSuccessors()[1] == return_block_); + + // Data flow. + header->AddInstruction(new (&allocator_) HIf(parameter_)); + body->AddInstruction(new (&allocator_) HGoto()); + + HPhi* phi = new (&allocator_) HPhi(&allocator_, 0, 0, Primitive::kPrimInt); + HInstruction* add = new (&allocator_) HAdd(Primitive::kPrimInt, phi, parameter_); + header->AddPhi(phi); + body->AddInstruction(add); + + phi->AddInput(add); + phi->AddInput(parameter_); + + graph_->ClearLoopInformation(); + graph_->ClearDominanceInformation(); + graph_->BuildDominatorTree(); + + // Check that after optimizations in BuildDominatorTree()/SimplifyCFG() phi inputs + // are still mapped correctly to the block predecessors. + for (size_t i = 0, e = phi->InputCount(); i < e; i++) { + HInstruction* input = phi->InputAt(i); + ASSERT_TRUE(input->GetBlock()->Dominates(header->GetPredecessors()[i])); + } +} } // namespace art diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 8644f676e8..508027e52a 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -358,6 +358,35 @@ void HGraph::SplitCriticalEdge(HBasicBlock* block, HBasicBlock* successor) { } } +// Reorder phi inputs to match reordering of the block's predecessors. +static void FixPhisAfterPredecessorsReodering(HBasicBlock* block, size_t first, size_t second) { + for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { + HPhi* phi = it.Current()->AsPhi(); + HInstruction* first_instr = phi->InputAt(first); + HInstruction* second_instr = phi->InputAt(second); + phi->ReplaceInput(first_instr, second); + phi->ReplaceInput(second_instr, first); + } +} + +// Make sure that the first predecessor of a loop header is the incoming block. +void HGraph::OrderLoopHeaderPredecessors(HBasicBlock* header) { + DCHECK(header->IsLoopHeader()); + HLoopInformation* info = header->GetLoopInformation(); + if (info->IsBackEdge(*header->GetPredecessors()[0])) { + HBasicBlock* to_swap = header->GetPredecessors()[0]; + for (size_t pred = 1, e = header->GetPredecessors().size(); pred < e; ++pred) { + HBasicBlock* predecessor = header->GetPredecessors()[pred]; + if (!info->IsBackEdge(*predecessor)) { + header->predecessors_[pred] = to_swap; + header->predecessors_[0] = predecessor; + FixPhisAfterPredecessorsReodering(header, 0, pred); + break; + } + } + } +} + void HGraph::SimplifyLoop(HBasicBlock* header) { HLoopInformation* info = header->GetLoopInformation(); @@ -381,18 +410,7 @@ void HGraph::SimplifyLoop(HBasicBlock* header) { pre_header->AddSuccessor(header); } - // Make sure the first predecessor of a loop header is the incoming block. - if (info->IsBackEdge(*header->GetPredecessors()[0])) { - HBasicBlock* to_swap = header->GetPredecessors()[0]; - for (size_t pred = 1, e = header->GetPredecessors().size(); pred < e; ++pred) { - HBasicBlock* predecessor = header->GetPredecessors()[pred]; - if (!info->IsBackEdge(*predecessor)) { - header->predecessors_[pred] = to_swap; - header->predecessors_[0] = predecessor; - break; - } - } - } + OrderLoopHeaderPredecessors(header); HInstruction* first_instruction = header->GetFirstInstruction(); if (first_instruction != nullptr && first_instruction->IsSuspendCheck()) { diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index 29be8acc0d..68d6c2ebbc 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -418,6 +418,7 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { HBasicBlock* SplitEdge(HBasicBlock* block, HBasicBlock* successor); void SplitCriticalEdge(HBasicBlock* block, HBasicBlock* successor); + void OrderLoopHeaderPredecessors(HBasicBlock* header); void SimplifyLoop(HBasicBlock* header); int32_t GetNextInstructionId() { diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 435ca1cad4..426a1691f5 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -22,8 +22,6 @@ #include <stdint.h> -#include "android-base/strings.h" - #ifdef ART_ENABLE_CODEGEN_arm64 #include "instruction_simplifier_arm64.h" #endif @@ -1127,12 +1125,7 @@ Compiler* CreateOptimizingCompiler(CompilerDriver* driver) { bool IsCompilingWithCoreImage() { const std::string& image = Runtime::Current()->GetImageLocation(); - // TODO: This is under-approximating... - if (android::base::EndsWith(image, "core.art") || - android::base::EndsWith(image, "core-optimizing.art")) { - return true; - } - return false; + return CompilerDriver::IsCoreImageFilename(image); } bool EncodeArtMethodInInlineInfo(ArtMethod* method ATTRIBUTE_UNUSED) { @@ -1226,14 +1219,14 @@ bool OptimizingCompiler::JitCompile(Thread* self, uint8_t* stack_map_data = nullptr; uint8_t* method_info_data = nullptr; uint8_t* roots_data = nullptr; - code_cache->ReserveData(self, - stack_map_size, - method_info_size, - number_of_roots, - method, - &stack_map_data, - &method_info_data, - &roots_data); + uint32_t data_size = code_cache->ReserveData(self, + stack_map_size, + method_info_size, + number_of_roots, + method, + &stack_map_data, + &method_info_data, + &roots_data); if (stack_map_data == nullptr || roots_data == nullptr) { return false; } @@ -1254,6 +1247,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, codegen->GetFpuSpillMask(), code_allocator.GetMemory().data(), code_allocator.GetSize(), + data_size, osr, roots, codegen->GetGraph()->HasShouldDeoptimizeFlag(), diff --git a/compiler/optimizing/scheduler.cc b/compiler/optimizing/scheduler.cc index 5ad011d8f9..38cd51bef6 100644 --- a/compiler/optimizing/scheduler.cc +++ b/compiler/optimizing/scheduler.cc @@ -554,6 +554,14 @@ SchedulingNode* CriticalPathSchedulingNodeSelector::GetHigherPrioritySchedulingN } void HScheduler::Schedule(HGraph* graph) { + // We run lsa here instead of in a separate pass to better control whether we + // should run the analysis or not. + LoadStoreAnalysis lsa(graph); + if (!only_optimize_loop_blocks_ || graph->HasLoops()) { + lsa.Run(); + scheduling_graph_.SetHeapLocationCollector(lsa.GetHeapLocationCollector()); + } + for (HBasicBlock* block : graph->GetReversePostOrder()) { if (IsSchedulable(block)) { Schedule(block); @@ -566,14 +574,6 @@ void HScheduler::Schedule(HBasicBlock* block) { // Build the scheduling graph. scheduling_graph_.Clear(); - - // Only perform LSA/HeapLocation analysis on the basic block that - // is going to get instruction scheduled. - HeapLocationCollector heap_location_collector(block->GetGraph()); - heap_location_collector.VisitBasicBlock(block); - heap_location_collector.BuildAliasingMatrix(); - scheduling_graph_.SetHeapLocationCollector(heap_location_collector); - for (HBackwardInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { HInstruction* instruction = it.Current(); CHECK_EQ(instruction->GetBlock(), block) @@ -724,8 +724,8 @@ bool HScheduler::IsSchedulable(const HInstruction* instruction) const { instruction->IsClassTableGet() || instruction->IsCurrentMethod() || instruction->IsDivZeroCheck() || - instruction->IsInstanceFieldGet() || - instruction->IsInstanceFieldSet() || + (instruction->IsInstanceFieldGet() && !instruction->AsInstanceFieldGet()->IsVolatile()) || + (instruction->IsInstanceFieldSet() && !instruction->AsInstanceFieldSet()->IsVolatile()) || instruction->IsInstanceOf() || instruction->IsInvokeInterface() || instruction->IsInvokeStaticOrDirect() || @@ -741,14 +741,10 @@ bool HScheduler::IsSchedulable(const HInstruction* instruction) const { instruction->IsReturn() || instruction->IsReturnVoid() || instruction->IsSelect() || - instruction->IsStaticFieldGet() || - instruction->IsStaticFieldSet() || + (instruction->IsStaticFieldGet() && !instruction->AsStaticFieldGet()->IsVolatile()) || + (instruction->IsStaticFieldSet() && !instruction->AsStaticFieldSet()->IsVolatile()) || instruction->IsSuspendCheck() || - instruction->IsTypeConversion() || - instruction->IsUnresolvedInstanceFieldGet() || - instruction->IsUnresolvedInstanceFieldSet() || - instruction->IsUnresolvedStaticFieldGet() || - instruction->IsUnresolvedStaticFieldSet(); + instruction->IsTypeConversion(); } bool HScheduler::IsSchedulable(const HBasicBlock* block) const { diff --git a/compiler/optimizing/scheduler_arm.cc b/compiler/optimizing/scheduler_arm.cc index ea15790105..d6eb6e3c52 100644 --- a/compiler/optimizing/scheduler_arm.cc +++ b/compiler/optimizing/scheduler_arm.cc @@ -20,6 +20,7 @@ #include "code_generator_utils.h" #include "common_arm.h" #include "mirror/array-inl.h" +#include "mirror/string.h" namespace art { namespace arm { diff --git a/compiler/optimizing/scheduler_arm64.cc b/compiler/optimizing/scheduler_arm64.cc index f54d3f3de2..510619faf9 100644 --- a/compiler/optimizing/scheduler_arm64.cc +++ b/compiler/optimizing/scheduler_arm64.cc @@ -18,6 +18,7 @@ #include "code_generator_utils.h" #include "mirror/array-inl.h" +#include "mirror/string.h" namespace art { namespace arm64 { diff --git a/compiler/utils/arm/assembler_arm_vixl.cc b/compiler/utils/arm/assembler_arm_vixl.cc index af3b4474e3..9df1b7434a 100644 --- a/compiler/utils/arm/assembler_arm_vixl.cc +++ b/compiler/utils/arm/assembler_arm_vixl.cc @@ -82,6 +82,22 @@ void ArmVIXLAssembler::MaybeUnpoisonHeapReference(vixl32::Register reg) { } } +void ArmVIXLAssembler::GenerateMarkingRegisterCheck(vixl32::Register temp, int code) { + // The Marking Register is only used in the Baker read barrier configuration. + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + vixl32::Label mr_is_ok; + + // temp = self.tls32_.is.gc_marking + ___ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value())); + // Check that mr == self.tls32_.is.gc_marking. + ___ Cmp(mr, temp); + ___ B(eq, &mr_is_ok, /* far_target */ false); + ___ Bkpt(code); + ___ Bind(&mr_is_ok); +} + void ArmVIXLAssembler::LoadImmediate(vixl32::Register rd, int32_t value) { // TODO(VIXL): Implement this optimization in VIXL. if (!ShifterOperandCanAlwaysHold(value) && ShifterOperandCanAlwaysHold(~value)) { diff --git a/compiler/utils/arm/assembler_arm_vixl.h b/compiler/utils/arm/assembler_arm_vixl.h index 66b22ea87c..9c11fd3222 100644 --- a/compiler/utils/arm/assembler_arm_vixl.h +++ b/compiler/utils/arm/assembler_arm_vixl.h @@ -178,6 +178,7 @@ class ArmVIXLAssembler FINAL : public Assembler { // // Heap poisoning. // + // Poison a heap reference contained in `reg`. void PoisonHeapReference(vixl32::Register reg); // Unpoison a heap reference contained in `reg`. @@ -187,6 +188,15 @@ class ArmVIXLAssembler FINAL : public Assembler { // Unpoison a heap reference contained in `reg` if heap poisoning is enabled. void MaybeUnpoisonHeapReference(vixl32::Register reg); + // Emit code checking the status of the Marking Register, and aborting + // the program if MR does not match the value stored in the art::Thread + // object. + // + // Argument `temp` is used as a temporary register to generate code. + // Argument `code` is used to identify the different occurrences of + // MaybeGenerateMarkingRegisterCheck and is passed to the BKPT instruction. + void GenerateMarkingRegisterCheck(vixl32::Register temp, int code = 0); + void StoreToOffset(StoreOperandType type, vixl32::Register reg, vixl32::Register base, diff --git a/compiler/utils/arm64/assembler_arm64.cc b/compiler/utils/arm64/assembler_arm64.cc index 6ed0e9b670..d8a48a563c 100644 --- a/compiler/utils/arm64/assembler_arm64.cc +++ b/compiler/utils/arm64/assembler_arm64.cc @@ -158,6 +158,24 @@ void Arm64Assembler::MaybeUnpoisonHeapReference(Register reg) { } } +void Arm64Assembler::GenerateMarkingRegisterCheck(Register temp, int code) { + // The Marking Register is only used in the Baker read barrier configuration. + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + vixl::aarch64::Register mr = reg_x(MR); // Marking Register. + vixl::aarch64::Register tr = reg_x(TR); // Thread Register. + vixl::aarch64::Label mr_is_ok; + + // temp = self.tls32_.is.gc_marking + ___ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value())); + // Check that mr == self.tls32_.is.gc_marking. + ___ Cmp(mr.W(), temp); + ___ B(eq, &mr_is_ok); + ___ Brk(code); + ___ Bind(&mr_is_ok); +} + #undef ___ } // namespace arm64 diff --git a/compiler/utils/arm64/assembler_arm64.h b/compiler/utils/arm64/assembler_arm64.h index 5b8a34e56d..6b28363a8f 100644 --- a/compiler/utils/arm64/assembler_arm64.h +++ b/compiler/utils/arm64/assembler_arm64.h @@ -98,6 +98,15 @@ class Arm64Assembler FINAL : public Assembler { // Unpoison a heap reference contained in `reg` if heap poisoning is enabled. void MaybeUnpoisonHeapReference(vixl::aarch64::Register reg); + // Emit code checking the status of the Marking Register, and aborting + // the program if MR does not match the value stored in the art::Thread + // object. + // + // Argument `temp` is used as a temporary register to generate code. + // Argument `code` is used to identify the different occurrences of + // MaybeGenerateMarkingRegisterCheck and is passed to the BRK instruction. + void GenerateMarkingRegisterCheck(vixl::aarch64::Register temp, int code = 0); + void Bind(Label* label ATTRIBUTE_UNUSED) OVERRIDE { UNIMPLEMENTED(FATAL) << "Do not use Bind for ARM64"; } diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.cc b/compiler/utils/arm64/jni_macro_assembler_arm64.cc index bab84bea4c..9732b765a1 100644 --- a/compiler/utils/arm64/jni_macro_assembler_arm64.cc +++ b/compiler/utils/arm64/jni_macro_assembler_arm64.cc @@ -662,7 +662,7 @@ void Arm64JNIMacroAssembler::Bind(JNIMacroLabel* label) { ___ Bind(Arm64JNIMacroLabel::Cast(label)->AsArm64()); } -void Arm64JNIMacroAssembler::EmitExceptionPoll(Arm64Exception *exception) { +void Arm64JNIMacroAssembler::EmitExceptionPoll(Arm64Exception* exception) { UseScratchRegisterScope temps(asm_.GetVIXLAssembler()); temps.Exclude(reg_x(exception->scratch_.AsXRegister())); Register temp = temps.AcquireX(); diff --git a/compiler/utils/jni_macro_assembler.h b/compiler/utils/jni_macro_assembler.h index 59a1a48e20..a8ca1119e5 100644 --- a/compiler/utils/jni_macro_assembler.h +++ b/compiler/utils/jni_macro_assembler.h @@ -216,8 +216,15 @@ class JNIMacroAssembler : public DeletableArenaObject<kArenaAllocAssembler> { */ virtual DebugFrameOpCodeWriterForAssembler& cfi() = 0; + void SetEmitRunTimeChecksInDebugMode(bool value) { + emit_run_time_checks_in_debug_mode_ = value; + } + protected: - explicit JNIMacroAssembler() {} + JNIMacroAssembler() {} + + // Should run-time checks be emitted in debug mode? + bool emit_run_time_checks_in_debug_mode_ = false; }; // A "Label" class used with the JNIMacroAssembler diff --git a/compiler/utils/x86_64/assembler_x86_64_test.cc b/compiler/utils/x86_64/assembler_x86_64_test.cc index ec14e7a825..9f2c44d30d 100644 --- a/compiler/utils/x86_64/assembler_x86_64_test.cc +++ b/compiler/utils/x86_64/assembler_x86_64_test.cc @@ -2012,7 +2012,7 @@ std::string buildframe_test_fn(JNIMacroAssemblerX86_64Test::Base* assembler_test x86_64::X86_64ManagedRegister method_reg = ManagedFromCpu(x86_64::RDI); size_t frame_size = 10 * kStackAlignment; - assembler->BuildFrame(10 * kStackAlignment, method_reg, spill_regs, entry_spills); + assembler->BuildFrame(frame_size, method_reg, spill_regs, entry_spills); // Construct assembly text counterpart. std::ostringstream str; @@ -2048,7 +2048,7 @@ std::string removeframe_test_fn(JNIMacroAssemblerX86_64Test::Base* assembler_tes ArrayRef<const ManagedRegister> spill_regs(raw_spill_regs); size_t frame_size = 10 * kStackAlignment; - assembler->RemoveFrame(10 * kStackAlignment, spill_regs); + assembler->RemoveFrame(frame_size, spill_regs); // Construct assembly text counterpart. std::ostringstream str; diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index e0ad64946e..e9ec5faa7f 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -355,6 +355,9 @@ NO_RETURN static void Usage(const char* fmt, ...) { UsageError(""); UsageError(" --debuggable: Produce code debuggable with Java debugger."); UsageError(""); + UsageError(" --avoid-storing-invocation: Avoid storing the invocation args in the key value"); + UsageError(" store. Used to test determinism with different args."); + UsageError(""); UsageError(" --runtime-arg <argument>: used to specify various arguments for the runtime,"); UsageError(" such as initial heap size, maximum heap size, and verbose output."); UsageError(" Use a separate --runtime-arg switch for each argument."); @@ -611,6 +614,7 @@ class Dex2Oat FINAL { dump_passes_(false), dump_timing_(false), dump_slow_timing_(kIsDebugBuild), + avoid_storing_invocation_(false), swap_fd_(kInvalidFd), app_image_fd_(kInvalidFd), profile_file_fd_(kInvalidFd), @@ -769,6 +773,11 @@ class Dex2Oat FINAL { compiler_options_->boot_image_ = !image_filenames_.empty(); compiler_options_->app_image_ = app_image_fd_ != -1 || !app_image_file_name_.empty(); + if (IsBootImage() && image_filenames_.size() == 1) { + const std::string& boot_image_filename = image_filenames_[0]; + compiler_options_->core_image_ = CompilerDriver::IsCoreImageFilename(boot_image_filename); + } + if (IsAppImage() && IsBootImage()) { Usage("Can't have both --image and (--app-image-fd or --app-image-file)"); } @@ -1133,14 +1142,16 @@ class Dex2Oat FINAL { void InsertCompileOptions(int argc, char** argv) { std::ostringstream oss; - for (int i = 0; i < argc; ++i) { - if (i > 0) { - oss << ' '; + if (!avoid_storing_invocation_) { + for (int i = 0; i < argc; ++i) { + if (i > 0) { + oss << ' '; + } + oss << argv[i]; } - oss << argv[i]; + key_value_store_->Put(OatHeader::kDex2OatCmdLineKey, oss.str()); + oss.str(""); // Reset. } - key_value_store_->Put(OatHeader::kDex2OatCmdLineKey, oss.str()); - oss.str(""); // Reset. oss << kRuntimeISA; key_value_store_->Put(OatHeader::kDex2OatHostKey, oss.str()); key_value_store_->Put( @@ -1271,6 +1282,8 @@ class Dex2Oat FINAL { dump_passes_ = true; } else if (option == "--dump-stats") { dump_stats_ = true; + } else if (option == "--avoid-storing-invocation") { + avoid_storing_invocation_ = true; } else if (option.starts_with("--swap-file=")) { swap_file_name_ = option.substr(strlen("--swap-file=")).data(); } else if (option.starts_with("--swap-fd=")) { @@ -1308,7 +1321,7 @@ class Dex2Oat FINAL { } else if (option.starts_with("--class-loader-context=")) { class_loader_context_ = ClassLoaderContext::Create( option.substr(strlen("--class-loader-context=")).data()); - if (class_loader_context_== nullptr) { + if (class_loader_context_ == nullptr) { Usage("Option --class-loader-context has an incorrect format: %s", option.data()); } } else if (option.starts_with("--dirty-image-objects=")) { @@ -1576,20 +1589,12 @@ class Dex2Oat FINAL { } // Open dex files for class path. + if (class_loader_context_ == nullptr) { - // TODO(calin): Temporary workaround while we transition to use - // --class-loader-context instead of --runtime-arg -cp - if (runtime_->GetClassPathString().empty()) { - class_loader_context_ = std::unique_ptr<ClassLoaderContext>( - new ClassLoaderContext()); - } else { - std::string spec = runtime_->GetClassPathString() == OatFile::kSpecialSharedLibrary - ? OatFile::kSpecialSharedLibrary - : "PCL[" + runtime_->GetClassPathString() + "]"; - class_loader_context_ = ClassLoaderContext::Create(spec); - } + // If no context was specified use the default one (which is an empty PathClassLoader). + class_loader_context_ = std::unique_ptr<ClassLoaderContext>(ClassLoaderContext::Default()); } - CHECK(class_loader_context_ != nullptr); + DCHECK_EQ(oat_writers_.size(), 1u); // Note: Ideally we would reject context where the source dex files are also @@ -1774,13 +1779,11 @@ class Dex2Oat FINAL { bool ShouldCompileDexFilesIndividually() const { // Compile individually if we are not building an image, not using any compilation, and are // using multidex. - // This means extract, verify, and quicken will use the individual compilation mode (to reduce + // This means extract, verify, and quicken, will use the individual compilation mode (to reduce // RAM used by the compiler). - // TODO: Still do it for app images to get testing coverage. Note that this will generate empty - // app images. return !IsImage() && dex_files_.size() > 1 && - !CompilerFilter::IsAnyCompilationEnabled(compiler_options_->GetCompilerFilter()); + !CompilerFilter::IsAotCompilationEnabled(compiler_options_->GetCompilerFilter()); } // Set up and create the compiler driver and then invoke it to compile all the dex files. @@ -1856,6 +1859,16 @@ class Dex2Oat FINAL { profile_compilation_info_.get())); driver_->SetDexFilesForOatFile(dex_files_); + const bool compile_individually = ShouldCompileDexFilesIndividually(); + if (compile_individually) { + // Set the compiler driver in the callbacks so that we can avoid re-verification. This not + // only helps performance but also prevents reverifying quickened bytecodes. Attempting + // verify quickened bytecode causes verification failures. + // Only set the compiler filter if we are doing separate compilation since there is a bit + // of overhead when checking if a class was previously verified. + callbacks_->SetDoesClassUnloading(true, driver_.get()); + } + // Setup vdex for compilation. if (!DoEagerUnquickeningOfVdex() && input_vdex_file_ != nullptr) { callbacks_->SetVerifierDeps( @@ -1872,7 +1885,7 @@ class Dex2Oat FINAL { callbacks_->SetVerifierDeps(new verifier::VerifierDeps(dex_files_)); } // Invoke the compilation. - if (ShouldCompileDexFilesIndividually()) { + if (compile_individually) { CompileDexFilesIndividually(); // Return a null classloader since we already freed released it. return nullptr; @@ -2899,6 +2912,7 @@ class Dex2Oat FINAL { bool dump_passes_; bool dump_timing_; bool dump_slow_timing_; + bool avoid_storing_invocation_; std::string swap_file_name_; int swap_fd_; size_t min_dex_files_for_swap_ = kDefaultMinDexFilesForSwap; @@ -2965,6 +2979,8 @@ class ScopedGlobalRef { static dex2oat::ReturnCode CompileImage(Dex2Oat& dex2oat) { dex2oat.LoadClassProfileDescriptors(); + // Keep the class loader that was used for compilation live for the rest of the compilation + // process. ScopedGlobalRef class_loader(dex2oat.Compile()); if (!dex2oat.WriteOutputFiles()) { @@ -3013,6 +3029,8 @@ static dex2oat::ReturnCode CompileImage(Dex2Oat& dex2oat) { } static dex2oat::ReturnCode CompileApp(Dex2Oat& dex2oat) { + // Keep the class loader that was used for compilation live for the rest of the compilation + // process. ScopedGlobalRef class_loader(dex2oat.Compile()); if (!dex2oat.WriteOutputFiles()) { diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc index 12bceb3c1f..6a9d979d52 100644 --- a/dex2oat/dex2oat_test.cc +++ b/dex2oat/dex2oat_test.cc @@ -41,6 +41,7 @@ namespace art { static constexpr size_t kMaxMethodIds = 65535; +static constexpr bool kDebugArgs = false; using android::base::StringPrintf; @@ -55,7 +56,7 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { } protected: - int GenerateOdexForTestWithStatus(const std::string& dex_location, + int GenerateOdexForTestWithStatus(const std::vector<std::string>& dex_locations, const std::string& odex_location, CompilerFilter::Filter filter, std::string* error_msg, @@ -63,7 +64,10 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { bool use_fd = false) { std::unique_ptr<File> oat_file; std::vector<std::string> args; - args.push_back("--dex-file=" + dex_location); + // Add dex file args. + for (const std::string& dex_location : dex_locations) { + args.push_back("--dex-file=" + dex_location); + } if (use_fd) { oat_file.reset(OS::CreateEmptyFile(odex_location.c_str())); CHECK(oat_file != nullptr) << odex_location; @@ -93,7 +97,7 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { bool use_fd = false, std::function<void(const OatFile&)> check_oat = [](const OatFile&) {}) { std::string error_msg; - int status = GenerateOdexForTestWithStatus(dex_location, + int status = GenerateOdexForTestWithStatus({dex_location}, odex_location, filter, &error_msg, @@ -187,6 +191,14 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { CHECK(android_root != nullptr); argv.push_back("--android-root=" + std::string(android_root)); + if (kDebugArgs) { + std::string all_args; + for (const std::string& arg : argv) { + all_args += arg + " "; + } + LOG(ERROR) << all_args; + } + int link[2]; if (pipe(link) == -1) { @@ -951,7 +963,7 @@ class Dex2oatReturnCodeTest : public Dex2oatTest { Copy(GetTestDexFileName(), dex_location); std::string error_msg; - return GenerateOdexForTestWithStatus(dex_location, + return GenerateOdexForTestWithStatus({dex_location}, odex_location, CompilerFilter::kSpeed, &error_msg, @@ -1107,4 +1119,72 @@ TEST_F(Dex2oatClassLoaderContextTest, ChainContext) { RunTest(context.c_str(), expected_classpath_key.c_str(), true); } +class Dex2oatDeterminism : public Dex2oatTest {}; + +TEST_F(Dex2oatDeterminism, UnloadCompile) { + if (!kUseReadBarrier && + gc::kCollectorTypeDefault != gc::kCollectorTypeCMS && + gc::kCollectorTypeDefault != gc::kCollectorTypeMS) { + LOG(INFO) << "Test requires determinism support."; + return; + } + Runtime* const runtime = Runtime::Current(); + std::string out_dir = GetScratchDir(); + const std::string base_oat_name = out_dir + "/base.oat"; + const std::string base_vdex_name = out_dir + "/base.vdex"; + const std::string unload_oat_name = out_dir + "/unload.oat"; + const std::string unload_vdex_name = out_dir + "/unload.vdex"; + const std::string no_unload_oat_name = out_dir + "/nounload.oat"; + const std::string no_unload_vdex_name = out_dir + "/nounload.vdex"; + const std::string app_image_name = out_dir + "/unload.art"; + std::string error_msg; + const std::vector<gc::space::ImageSpace*>& spaces = runtime->GetHeap()->GetBootImageSpaces(); + ASSERT_GT(spaces.size(), 0u); + const std::string image_location = spaces[0]->GetImageLocation(); + // Without passing in an app image, it will unload in between compilations. + const int res = GenerateOdexForTestWithStatus( + GetLibCoreDexFileNames(), + base_oat_name, + CompilerFilter::Filter::kQuicken, + &error_msg, + {"--force-determinism", "--avoid-storing-invocation"}); + EXPECT_EQ(res, 0); + Copy(base_oat_name, unload_oat_name); + Copy(base_vdex_name, unload_vdex_name); + std::unique_ptr<File> unload_oat(OS::OpenFileForReading(unload_oat_name.c_str())); + std::unique_ptr<File> unload_vdex(OS::OpenFileForReading(unload_vdex_name.c_str())); + ASSERT_TRUE(unload_oat != nullptr); + ASSERT_TRUE(unload_vdex != nullptr); + EXPECT_GT(unload_oat->GetLength(), 0u); + EXPECT_GT(unload_vdex->GetLength(), 0u); + // Regenerate with an app image to disable the dex2oat unloading and verify that the output is + // the same. + const int res2 = GenerateOdexForTestWithStatus( + GetLibCoreDexFileNames(), + base_oat_name, + CompilerFilter::Filter::kQuicken, + &error_msg, + {"--force-determinism", "--avoid-storing-invocation", "--app-image-file=" + app_image_name}); + EXPECT_EQ(res2, 0); + Copy(base_oat_name, no_unload_oat_name); + Copy(base_vdex_name, no_unload_vdex_name); + std::unique_ptr<File> no_unload_oat(OS::OpenFileForReading(no_unload_oat_name.c_str())); + std::unique_ptr<File> no_unload_vdex(OS::OpenFileForReading(no_unload_vdex_name.c_str())); + ASSERT_TRUE(no_unload_oat != nullptr); + ASSERT_TRUE(no_unload_vdex != nullptr); + EXPECT_GT(no_unload_oat->GetLength(), 0u); + EXPECT_GT(no_unload_vdex->GetLength(), 0u); + // Verify that both of the files are the same (odex and vdex). + EXPECT_EQ(unload_oat->GetLength(), no_unload_oat->GetLength()); + EXPECT_EQ(unload_vdex->GetLength(), no_unload_vdex->GetLength()); + EXPECT_EQ(unload_oat->Compare(no_unload_oat.get()), 0) + << unload_oat_name << " " << no_unload_oat_name; + EXPECT_EQ(unload_vdex->Compare(no_unload_vdex.get()), 0) + << unload_vdex_name << " " << no_unload_vdex_name; + // App image file. + std::unique_ptr<File> app_image_file(OS::OpenFileForReading(app_image_name.c_str())); + ASSERT_TRUE(app_image_file != nullptr); + EXPECT_GT(app_image_file->GetLength(), 0u); +} + } // namespace art diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc index 99168c9bc5..f8b1f5375a 100644 --- a/oatdump/oatdump.cc +++ b/oatdump/oatdump.cc @@ -89,6 +89,8 @@ const char* image_methods_descriptions_[] = { "kSaveRefsOnlyMethod", "kSaveRefsAndArgsMethod", "kSaveEverythingMethod", + "kSaveEverythingMethodForClinit", + "kSaveEverythingMethodForSuspendCheck", }; const char* image_roots_descriptions_[] = { diff --git a/runtime/aot_class_linker.cc b/runtime/aot_class_linker.cc index b1bc3f8f2e..9396565eee 100644 --- a/runtime/aot_class_linker.cc +++ b/runtime/aot_class_linker.cc @@ -16,10 +16,12 @@ #include "aot_class_linker.h" +#include "class_reference.h" +#include "compiler_callbacks.h" #include "handle_scope-inl.h" -#include "mirror/class.h" -#include "mirror/object-inl.h" +#include "mirror/class-inl.h" #include "runtime.h" +#include "verifier/verifier_enums.h" namespace art { @@ -66,4 +68,18 @@ bool AotClassLinker::InitializeClass(Thread* self, Handle<mirror::Class> klass, } return success; } + +verifier::FailureKind AotClassLinker::PerformClassVerification(Thread* self, + Handle<mirror::Class> klass, + verifier::HardFailLogMode log_level, + std::string* error_msg) { + Runtime* const runtime = Runtime::Current(); + CompilerCallbacks* callbacks = runtime->GetCompilerCallbacks(); + if (callbacks->CanAssumeVerified(ClassReference(&klass->GetDexFile(), + klass->GetDexClassDefIndex()))) { + return verifier::FailureKind::kNoFailure; + } + return ClassLinker::PerformClassVerification(self, klass, log_level, error_msg); +} + } // namespace art diff --git a/runtime/aot_class_linker.h b/runtime/aot_class_linker.h index 11bea86fc4..927b53302b 100644 --- a/runtime/aot_class_linker.h +++ b/runtime/aot_class_linker.h @@ -27,6 +27,16 @@ class AotClassLinker : public ClassLinker { explicit AotClassLinker(InternTable *intern_table); ~AotClassLinker(); + protected: + // Overridden version of PerformClassVerification allows skipping verification if the class was + // previously verified but unloaded. + verifier::FailureKind PerformClassVerification(Thread* self, + Handle<mirror::Class> klass, + verifier::HardFailLogMode log_level, + std::string* error_msg) + OVERRIDE + REQUIRES_SHARED(Locks::mutator_lock_); + bool InitializeClass(Thread *self, Handle<mirror::Class> klass, bool can_run_clinit, diff --git a/runtime/arch/arch_test.cc b/runtime/arch/arch_test.cc index bfac8c422d..1ba4070056 100644 --- a/runtime/arch/arch_test.cc +++ b/runtime/arch/arch_test.cc @@ -88,6 +88,11 @@ static constexpr size_t kFrameSizeSaveRefsOnly = FRAME_SIZE_SAVE_REFS_ONLY; #undef FRAME_SIZE_SAVE_REFS_ONLY static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS; #undef FRAME_SIZE_SAVE_REFS_AND_ARGS +static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT +static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck = + FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING; #undef FRAME_SIZE_SAVE_EVERYTHING #undef BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET @@ -109,6 +114,11 @@ static constexpr size_t kFrameSizeSaveRefsOnly = FRAME_SIZE_SAVE_REFS_ONLY; #undef FRAME_SIZE_SAVE_REFS_ONLY static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS; #undef FRAME_SIZE_SAVE_REFS_AND_ARGS +static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT +static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck = + FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING; #undef FRAME_SIZE_SAVE_EVERYTHING #undef BAKER_MARK_INTROSPECTION_ARRAY_SWITCH_OFFSET @@ -126,6 +136,11 @@ static constexpr size_t kFrameSizeSaveRefsOnly = FRAME_SIZE_SAVE_REFS_ONLY; #undef FRAME_SIZE_SAVE_REFS_ONLY static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS; #undef FRAME_SIZE_SAVE_REFS_AND_ARGS +static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT +static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck = + FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING; #undef FRAME_SIZE_SAVE_EVERYTHING #undef BAKER_MARK_INTROSPECTION_REGISTER_COUNT @@ -142,6 +157,11 @@ static constexpr size_t kFrameSizeSaveRefsOnly = FRAME_SIZE_SAVE_REFS_ONLY; #undef FRAME_SIZE_SAVE_REFS_ONLY static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS; #undef FRAME_SIZE_SAVE_REFS_AND_ARGS +static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT +static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck = + FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING; #undef FRAME_SIZE_SAVE_EVERYTHING #undef BAKER_MARK_INTROSPECTION_REGISTER_COUNT @@ -158,6 +178,11 @@ static constexpr size_t kFrameSizeSaveRefsOnly = FRAME_SIZE_SAVE_REFS_ONLY; #undef FRAME_SIZE_SAVE_REFS_ONLY static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS; #undef FRAME_SIZE_SAVE_REFS_AND_ARGS +static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT +static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck = + FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING; #undef FRAME_SIZE_SAVE_EVERYTHING } // namespace x86 @@ -170,25 +195,36 @@ static constexpr size_t kFrameSizeSaveRefsOnly = FRAME_SIZE_SAVE_REFS_ONLY; #undef FRAME_SIZE_SAVE_REFS_ONLY static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS; #undef FRAME_SIZE_SAVE_REFS_AND_ARGS +static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT +static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck = + FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK; +#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING; #undef FRAME_SIZE_SAVE_EVERYTHING } // namespace x86_64 // Check architecture specific constants are sound. -#define TEST_ARCH(Arch, arch) \ - TEST_F(ArchTest, Arch) { \ - CheckFrameSize(InstructionSet::k##Arch, \ - CalleeSaveType::kSaveAllCalleeSaves, \ - arch::kFrameSizeSaveAllCalleeSaves); \ - CheckFrameSize(InstructionSet::k##Arch, \ - CalleeSaveType::kSaveRefsOnly, \ - arch::kFrameSizeSaveRefsOnly); \ - CheckFrameSize(InstructionSet::k##Arch, \ - CalleeSaveType::kSaveRefsAndArgs, \ - arch::kFrameSizeSaveRefsAndArgs); \ - CheckFrameSize(InstructionSet::k##Arch, \ - CalleeSaveType::kSaveEverything, \ - arch::kFrameSizeSaveEverything); \ +#define TEST_ARCH(Arch, arch) \ + TEST_F(ArchTest, Arch) { \ + CheckFrameSize(InstructionSet::k##Arch, \ + CalleeSaveType::kSaveAllCalleeSaves, \ + arch::kFrameSizeSaveAllCalleeSaves); \ + CheckFrameSize(InstructionSet::k##Arch, \ + CalleeSaveType::kSaveRefsOnly, \ + arch::kFrameSizeSaveRefsOnly); \ + CheckFrameSize(InstructionSet::k##Arch, \ + CalleeSaveType::kSaveRefsAndArgs, \ + arch::kFrameSizeSaveRefsAndArgs); \ + CheckFrameSize(InstructionSet::k##Arch, \ + CalleeSaveType::kSaveEverything, \ + arch::kFrameSizeSaveEverything); \ + CheckFrameSize(InstructionSet::k##Arch, \ + CalleeSaveType::kSaveEverythingForClinit, \ + arch::kFrameSizeSaveEverythingForClinit); \ + CheckFrameSize(InstructionSet::k##Arch, \ + CalleeSaveType::kSaveEverythingForSuspendCheck, \ + arch::kFrameSizeSaveEverythingForSuspendCheck); \ } TEST_ARCH(Arm, arm) TEST_ARCH(Arm64, arm64) diff --git a/runtime/arch/arm/asm_support_arm.h b/runtime/arch/arm/asm_support_arm.h index 8f2fd6ecc9..3d85872617 100644 --- a/runtime/arch/arm/asm_support_arm.h +++ b/runtime/arch/arm/asm_support_arm.h @@ -23,6 +23,8 @@ #define FRAME_SIZE_SAVE_REFS_ONLY 32 #define FRAME_SIZE_SAVE_REFS_AND_ARGS 112 #define FRAME_SIZE_SAVE_EVERYTHING 192 +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING // The offset from the art_quick_read_barrier_mark_introspection (used for field // loads with 32-bit LDR) to the entrypoint for field loads with 16-bit LDR, diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index 375768ec3f..cf2bfeeeef 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -182,7 +182,7 @@ * Runtime::CreateCalleeSaveMethod(kSaveEverything) * when core registers are already saved. */ -.macro SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED rTemp +.macro SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED rTemp, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET @ 14 words of callee saves and args already saved. vpush {d0-d15} @ 32 words, 2 for each of the 16 saved doubles. .cfi_adjust_cfa_offset 128 @@ -190,7 +190,7 @@ .cfi_adjust_cfa_offset 8 RUNTIME_CURRENT1 \rTemp @ Load Runtime::Current into rTemp. @ Load kSaveEverything Method* into rTemp. - ldr \rTemp, [\rTemp, #RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET] + ldr \rTemp, [\rTemp, #\runtime_method_offset] str \rTemp, [sp, #0] @ Place Method* at bottom of stack. str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. @@ -204,7 +204,7 @@ * Macro that sets up the callee save frame to conform with * Runtime::CreateCalleeSaveMethod(kSaveEverything) */ -.macro SETUP_SAVE_EVERYTHING_FRAME rTemp +.macro SETUP_SAVE_EVERYTHING_FRAME rTemp, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET push {r0-r12, lr} @ 14 words of callee saves and args. .cfi_adjust_cfa_offset 56 .cfi_rel_offset r0, 0 @@ -221,7 +221,7 @@ .cfi_rel_offset r11, 44 .cfi_rel_offset ip, 48 .cfi_rel_offset lr, 52 - SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED \rTemp + SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED \rTemp, \runtime_method_offset .endm .macro RESTORE_SAVE_EVERYTHING_FRAME @@ -1004,10 +1004,10 @@ END \name .endm // Macro for string and type resolution and initialization. -.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint +.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET .extern \entrypoint ENTRY \name - SETUP_SAVE_EVERYTHING_FRAME r1 @ save everything in case of GC + SETUP_SAVE_EVERYTHING_FRAME r1, \runtime_method_offset @ save everything in case of GC mov r1, r9 @ pass Thread::Current bl \entrypoint @ (uint32_t index, Thread*) cbz r0, 1f @ If result is null, deliver the OOME. @@ -1021,8 +1021,12 @@ ENTRY \name END \name .endm -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode +.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint + ONE_ARG_SAVE_EVERYTHING_DOWNCALL \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET +.endm + +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode @@ -1537,7 +1541,7 @@ ENTRY art_quick_test_suspend 1: mov rSUSPEND, #SUSPEND_CHECK_INTERVAL @ reset rSUSPEND to SUSPEND_CHECK_INTERVAL #endif - SETUP_SAVE_EVERYTHING_FRAME r0 @ save everything for GC stack crawl + SETUP_SAVE_EVERYTHING_FRAME r0, RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET @ save everything for GC stack crawl mov r0, rSELF bl artTestSuspendFromCode @ (Thread*) RESTORE_SAVE_EVERYTHING_FRAME diff --git a/runtime/arch/arm/quick_method_frame_info_arm.h b/runtime/arch/arm/quick_method_frame_info_arm.h index 39061f0d4d..5c5b81baae 100644 --- a/runtime/arch/arm/quick_method_frame_info_arm.h +++ b/runtime/arch/arm/quick_method_frame_info_arm.h @@ -56,6 +56,7 @@ static constexpr uint32_t kArmCalleeSaveFpEverythingSpills = kArmCalleeSaveFpArgSpills | kArmCalleeSaveFpAllSpills; constexpr uint32_t ArmCalleeSaveCoreSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kArmCalleeSaveAlwaysSpills | kArmCalleeSaveRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kArmCalleeSaveArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kArmCalleeSaveAllSpills : 0) | @@ -63,6 +64,7 @@ constexpr uint32_t ArmCalleeSaveCoreSpills(CalleeSaveType type) { } constexpr uint32_t ArmCalleeSaveFpSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kArmCalleeSaveFpAlwaysSpills | kArmCalleeSaveFpRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kArmCalleeSaveFpArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kArmCalleeSaveFpAllSpills : 0) | @@ -70,29 +72,34 @@ constexpr uint32_t ArmCalleeSaveFpSpills(CalleeSaveType type) { } constexpr uint32_t ArmCalleeSaveFrameSize(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return RoundUp((POPCOUNT(ArmCalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(ArmCalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kArmPointerSize), kStackAlignment); } constexpr QuickMethodFrameInfo ArmCalleeSaveMethodFrameInfo(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return QuickMethodFrameInfo(ArmCalleeSaveFrameSize(type), ArmCalleeSaveCoreSpills(type), ArmCalleeSaveFpSpills(type)); } constexpr size_t ArmCalleeSaveFpr1Offset(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return ArmCalleeSaveFrameSize(type) - (POPCOUNT(ArmCalleeSaveCoreSpills(type)) + POPCOUNT(ArmCalleeSaveFpSpills(type))) * static_cast<size_t>(kArmPointerSize); } constexpr size_t ArmCalleeSaveGpr1Offset(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return ArmCalleeSaveFrameSize(type) - POPCOUNT(ArmCalleeSaveCoreSpills(type)) * static_cast<size_t>(kArmPointerSize); } constexpr size_t ArmCalleeSaveLrOffset(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return ArmCalleeSaveFrameSize(type) - POPCOUNT(ArmCalleeSaveCoreSpills(type) & (-(1 << LR))) * static_cast<size_t>(kArmPointerSize); } diff --git a/runtime/arch/arm64/asm_support_arm64.h b/runtime/arch/arm64/asm_support_arm64.h index 6b7720023e..57376d0c4f 100644 --- a/runtime/arch/arm64/asm_support_arm64.h +++ b/runtime/arch/arm64/asm_support_arm64.h @@ -23,6 +23,8 @@ #define FRAME_SIZE_SAVE_REFS_ONLY 96 #define FRAME_SIZE_SAVE_REFS_AND_ARGS 224 #define FRAME_SIZE_SAVE_EVERYTHING 512 +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING // The offset from art_quick_read_barrier_mark_introspection to the array switch cases, // i.e. art_quick_read_barrier_mark_introspection_arrays. diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index d15f5b85ec..3d8ca402cf 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -287,7 +287,7 @@ * when the SP has already been decremented by FRAME_SIZE_SAVE_EVERYTHING * and saving registers x29 and LR is handled elsewhere. */ -.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR +.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET // Ugly compile-time check, but we only have the preprocessor. #if (FRAME_SIZE_SAVE_EVERYTHING != 512) #error "FRAME_SIZE_SAVE_EVERYTHING(ARM64) size not as expected." @@ -337,7 +337,7 @@ ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveEverything]; - ldr xIP0, [xIP0, RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET] + ldr xIP0, [xIP0, \runtime_method_offset] // Store ArtMethod* Runtime::callee_save_methods_[kSaveEverything]. str xIP0, [sp] @@ -350,10 +350,10 @@ * Macro that sets up the callee save frame to conform with * Runtime::CreateCalleeSaveMethod(kSaveEverything) */ -.macro SETUP_SAVE_EVERYTHING_FRAME +.macro SETUP_SAVE_EVERYTHING_FRAME runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET INCREASE_FRAME 512 SAVE_TWO_REGS x29, xLR, 496 - SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR + SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR \runtime_method_offset .endm .macro RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 @@ -398,7 +398,7 @@ .endm .macro RESTORE_SAVE_EVERYTHING_FRAME - RESTORE_REG x0, 264 + RESTORE_REG x0, 264 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 .endm @@ -1593,10 +1593,10 @@ END \name .endm // Macro for string and type resolution and initialization. -.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint +.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET .extern \entrypoint ENTRY \name - SETUP_SAVE_EVERYTHING_FRAME // save everything for stack crawl + SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for stack crawl mov x1, xSELF // pass Thread::Current bl \entrypoint // (int32_t index, Thread* self) cbz w0, 1f // If result is null, deliver the OOME. @@ -1611,6 +1611,10 @@ ENTRY \name END \name .endm +.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint + ONE_ARG_SAVE_EVERYTHING_DOWNCALL \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET +.endm + .macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER cbz w0, 1f // result zero branch over ret // return @@ -1629,9 +1633,8 @@ TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, * initializer and deliver the exception on error. On success the static storage base is * returned. */ -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode - -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode @@ -2008,7 +2011,7 @@ GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFr */ .extern artTestSuspendFromCode ENTRY art_quick_test_suspend - SETUP_SAVE_EVERYTHING_FRAME // save callee saves for stack crawl + SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save callee saves for stack crawl mov x0, xSELF bl artTestSuspendFromCode // (Thread*) RESTORE_SAVE_EVERYTHING_FRAME diff --git a/runtime/arch/arm64/quick_method_frame_info_arm64.h b/runtime/arch/arm64/quick_method_frame_info_arm64.h index c231d4d3d4..3e6f6c6e3b 100644 --- a/runtime/arch/arm64/quick_method_frame_info_arm64.h +++ b/runtime/arch/arm64/quick_method_frame_info_arm64.h @@ -80,6 +80,7 @@ static constexpr uint32_t kArm64CalleeSaveFpEverythingSpills = (1 << art::arm64::D30) | (1 << art::arm64::D31); constexpr uint32_t Arm64CalleeSaveCoreSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kArm64CalleeSaveAlwaysSpills | kArm64CalleeSaveRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kArm64CalleeSaveArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kArm64CalleeSaveAllSpills : 0) | @@ -87,6 +88,7 @@ constexpr uint32_t Arm64CalleeSaveCoreSpills(CalleeSaveType type) { } constexpr uint32_t Arm64CalleeSaveFpSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kArm64CalleeSaveFpAlwaysSpills | kArm64CalleeSaveFpRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kArm64CalleeSaveFpArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kArm64CalleeSaveFpAllSpills : 0) | @@ -94,29 +96,34 @@ constexpr uint32_t Arm64CalleeSaveFpSpills(CalleeSaveType type) { } constexpr uint32_t Arm64CalleeSaveFrameSize(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return RoundUp((POPCOUNT(Arm64CalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(Arm64CalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kArm64PointerSize), kStackAlignment); } constexpr QuickMethodFrameInfo Arm64CalleeSaveMethodFrameInfo(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return QuickMethodFrameInfo(Arm64CalleeSaveFrameSize(type), Arm64CalleeSaveCoreSpills(type), Arm64CalleeSaveFpSpills(type)); } constexpr size_t Arm64CalleeSaveFpr1Offset(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return Arm64CalleeSaveFrameSize(type) - (POPCOUNT(Arm64CalleeSaveCoreSpills(type)) + POPCOUNT(Arm64CalleeSaveFpSpills(type))) * static_cast<size_t>(kArm64PointerSize); } constexpr size_t Arm64CalleeSaveGpr1Offset(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return Arm64CalleeSaveFrameSize(type) - POPCOUNT(Arm64CalleeSaveCoreSpills(type)) * static_cast<size_t>(kArm64PointerSize); } constexpr size_t Arm64CalleeSaveLrOffset(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return Arm64CalleeSaveFrameSize(type) - POPCOUNT(Arm64CalleeSaveCoreSpills(type) & (-(1 << LR))) * static_cast<size_t>(kArm64PointerSize); diff --git a/runtime/arch/mips/asm_support_mips.h b/runtime/arch/mips/asm_support_mips.h index 9d8572ffb5..2edd63f58a 100644 --- a/runtime/arch/mips/asm_support_mips.h +++ b/runtime/arch/mips/asm_support_mips.h @@ -23,6 +23,8 @@ #define FRAME_SIZE_SAVE_REFS_ONLY 48 #define FRAME_SIZE_SAVE_REFS_AND_ARGS 112 #define FRAME_SIZE_SAVE_EVERYTHING 256 +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING // &art_quick_read_barrier_mark_introspection is the first of many entry points: // 21 entry points for long field offsets, large array indices and variable array indices diff --git a/runtime/arch/mips/quick_method_frame_info_mips.h b/runtime/arch/mips/quick_method_frame_info_mips.h index 01879a5cea..45a21ab942 100644 --- a/runtime/arch/mips/quick_method_frame_info_mips.h +++ b/runtime/arch/mips/quick_method_frame_info_mips.h @@ -65,6 +65,7 @@ static constexpr uint32_t kMipsCalleeSaveFpEverythingSpills = (1 << art::mips::F28) | (1 << art::mips::F29) | (1 << art::mips::F30) | (1u << art::mips::F31); constexpr uint32_t MipsCalleeSaveCoreSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kMipsCalleeSaveAlwaysSpills | kMipsCalleeSaveRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kMipsCalleeSaveArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kMipsCalleeSaveAllSpills : 0) | @@ -72,6 +73,7 @@ constexpr uint32_t MipsCalleeSaveCoreSpills(CalleeSaveType type) { } constexpr uint32_t MipsCalleeSaveFPSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kMipsCalleeSaveFpAlwaysSpills | kMipsCalleeSaveFpRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kMipsCalleeSaveFpArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kMipsCalleeSaveAllFPSpills : 0) | @@ -79,12 +81,14 @@ constexpr uint32_t MipsCalleeSaveFPSpills(CalleeSaveType type) { } constexpr uint32_t MipsCalleeSaveFrameSize(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return RoundUp((POPCOUNT(MipsCalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(MipsCalleeSaveFPSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kMipsPointerSize), kStackAlignment); } constexpr QuickMethodFrameInfo MipsCalleeSaveMethodFrameInfo(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return QuickMethodFrameInfo(MipsCalleeSaveFrameSize(type), MipsCalleeSaveCoreSpills(type), MipsCalleeSaveFPSpills(type)); diff --git a/runtime/arch/mips64/asm_support_mips64.h b/runtime/arch/mips64/asm_support_mips64.h index 7185da550c..a8e907eca5 100644 --- a/runtime/arch/mips64/asm_support_mips64.h +++ b/runtime/arch/mips64/asm_support_mips64.h @@ -27,6 +27,8 @@ #define FRAME_SIZE_SAVE_REFS_AND_ARGS 208 // $f0-$f31, $at, $v0-$v1, $a0-$a7, $t0-$t3, $s0-$s7, $t8-$t9, $gp, $s8, $ra + padding + method* #define FRAME_SIZE_SAVE_EVERYTHING 496 +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING // &art_quick_read_barrier_mark_introspection is the first of many entry points: // 20 entry points for long field offsets, large array indices and variable array indices diff --git a/runtime/arch/mips64/quick_method_frame_info_mips64.h b/runtime/arch/mips64/quick_method_frame_info_mips64.h index a55ab0e196..520f6319d5 100644 --- a/runtime/arch/mips64/quick_method_frame_info_mips64.h +++ b/runtime/arch/mips64/quick_method_frame_info_mips64.h @@ -72,6 +72,7 @@ static constexpr uint32_t kMips64CalleeSaveFpEverythingSpills = (1 << art::mips64::F30) | (1 << art::mips64::F31); constexpr uint32_t Mips64CalleeSaveCoreSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kMips64CalleeSaveAlwaysSpills | kMips64CalleeSaveRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kMips64CalleeSaveArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kMips64CalleeSaveAllSpills : 0) | @@ -79,6 +80,7 @@ constexpr uint32_t Mips64CalleeSaveCoreSpills(CalleeSaveType type) { } constexpr uint32_t Mips64CalleeSaveFpSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kMips64CalleeSaveFpRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kMips64CalleeSaveFpArgSpills : 0) | (type == CalleeSaveType::kSaveAllCalleeSaves ? kMips64CalleeSaveFpAllSpills : 0) | @@ -86,12 +88,14 @@ constexpr uint32_t Mips64CalleeSaveFpSpills(CalleeSaveType type) { } constexpr uint32_t Mips64CalleeSaveFrameSize(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return RoundUp((POPCOUNT(Mips64CalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(Mips64CalleeSaveFpSpills(type)) /* fprs */ + + 1 /* Method* */) * static_cast<size_t>(kMips64PointerSize), kStackAlignment); } constexpr QuickMethodFrameInfo Mips64CalleeSaveMethodFrameInfo(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return QuickMethodFrameInfo(Mips64CalleeSaveFrameSize(type), Mips64CalleeSaveCoreSpills(type), Mips64CalleeSaveFpSpills(type)); diff --git a/runtime/arch/x86/asm_support_x86.h b/runtime/arch/x86/asm_support_x86.h index 2bba08d571..737d736f01 100644 --- a/runtime/arch/x86/asm_support_x86.h +++ b/runtime/arch/x86/asm_support_x86.h @@ -23,5 +23,7 @@ #define FRAME_SIZE_SAVE_REFS_ONLY 32 #define FRAME_SIZE_SAVE_REFS_AND_ARGS (32 + 32) #define FRAME_SIZE_SAVE_EVERYTHING (48 + 64) +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING #endif // ART_RUNTIME_ARCH_X86_ASM_SUPPORT_X86_H_ diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S index 48d2de9567..4e5e93ac5a 100644 --- a/runtime/arch/x86/quick_entrypoints_x86.S +++ b/runtime/arch/x86/quick_entrypoints_x86.S @@ -226,7 +226,7 @@ END_MACRO * Runtime::CreateCalleeSaveMethod(kSaveEverything) * when EDI and ESI are already saved. */ -MACRO2(SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED, got_reg, temp_reg) +MACRO3(SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED, got_reg, temp_reg, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) // Save core registers from highest to lowest to agree with core spills bitmap. // EDI and ESI, or at least placeholders for them, are already on the stack. PUSH ebp @@ -252,7 +252,7 @@ MACRO2(SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED, got_reg, temp_reg) movl SYMBOL(_ZN3art7Runtime9instance_E)@GOT(REG_VAR(got_reg)), REG_VAR(temp_reg) movl (REG_VAR(temp_reg)), REG_VAR(temp_reg) // Push save everything callee-save method. - pushl RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET(REG_VAR(temp_reg)) + pushl \runtime_method_offset(REG_VAR(temp_reg)) CFI_ADJUST_CFA_OFFSET(4) // Store esp as the stop quick frame. movl %esp, %fs:THREAD_TOP_QUICK_FRAME_OFFSET @@ -269,20 +269,20 @@ END_MACRO * Runtime::CreateCalleeSaveMethod(kSaveEverything) * when EDI is already saved. */ -MACRO2(SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED, got_reg, temp_reg) +MACRO3(SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED, got_reg, temp_reg, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) // Save core registers from highest to lowest to agree with core spills bitmap. // EDI, or at least a placeholder for it, is already on the stack. PUSH esi - SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg) + SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg), \runtime_method_offset END_MACRO /* * Macro that sets up the callee save frame to conform with * Runtime::CreateCalleeSaveMethod(kSaveEverything) */ -MACRO2(SETUP_SAVE_EVERYTHING_FRAME, got_reg, temp_reg) +MACRO3(SETUP_SAVE_EVERYTHING_FRAME, got_reg, temp_reg, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) PUSH edi - SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg) + SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg), \runtime_method_offset END_MACRO MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_FRPS) @@ -923,9 +923,9 @@ MACRO3(THREE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) END_MACRO // Macro for string and type resolution and initialization. -MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name) +MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) DEFINE_FUNCTION VAR(c_name) - SETUP_SAVE_EVERYTHING_FRAME ebx, ebx // save ref containing registers for GC + SETUP_SAVE_EVERYTHING_FRAME ebx, ebx, \runtime_method_offset // save ref containing registers for GC // Outgoing argument set up subl MACRO_LITERAL(8), %esp // push padding CFI_ADJUST_CFA_OFFSET(8) @@ -947,6 +947,10 @@ MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name) END_FUNCTION VAR(c_name) END_MACRO +MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name) + ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET +END_MACRO + MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER) testl %eax, %eax // eax == 0 ? jz 1f // if eax == 0 goto 1 @@ -1270,8 +1274,8 @@ GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, artAllocArrayFr GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_32 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_64 -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode @@ -1598,7 +1602,7 @@ DEFINE_FUNCTION art_quick_memcpy END_FUNCTION art_quick_memcpy DEFINE_FUNCTION art_quick_test_suspend - SETUP_SAVE_EVERYTHING_FRAME ebx, ebx // save everything for GC + SETUP_SAVE_EVERYTHING_FRAME ebx, ebx, RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save everything for GC // Outgoing argument set up subl MACRO_LITERAL(12), %esp // push padding CFI_ADJUST_CFA_OFFSET(12) diff --git a/runtime/arch/x86/quick_method_frame_info_x86.h b/runtime/arch/x86/quick_method_frame_info_x86.h index 8342c9fe03..9a6633365c 100644 --- a/runtime/arch/x86/quick_method_frame_info_x86.h +++ b/runtime/arch/x86/quick_method_frame_info_x86.h @@ -57,23 +57,27 @@ static constexpr uint32_t kX86CalleeSaveFpEverythingSpills = (1 << art::x86::XMM6) | (1 << art::x86::XMM7); constexpr uint32_t X86CalleeSaveCoreSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kX86CalleeSaveAlwaysSpills | kX86CalleeSaveRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveArgSpills : 0) | (type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveEverythingSpills : 0); } constexpr uint32_t X86CalleeSaveFpSpills(CalleeSaveType type) { - return (type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveFpArgSpills : 0) | - (type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveFpEverythingSpills : 0); + type = GetCanonicalCalleeSaveType(type); + return (type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveFpArgSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveFpEverythingSpills : 0); } constexpr uint32_t X86CalleeSaveFrameSize(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return RoundUp((POPCOUNT(X86CalleeSaveCoreSpills(type)) /* gprs */ + 2 * POPCOUNT(X86CalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kX86PointerSize), kStackAlignment); } constexpr QuickMethodFrameInfo X86CalleeSaveMethodFrameInfo(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return QuickMethodFrameInfo(X86CalleeSaveFrameSize(type), X86CalleeSaveCoreSpills(type), X86CalleeSaveFpSpills(type)); diff --git a/runtime/arch/x86_64/asm_support_x86_64.h b/runtime/arch/x86_64/asm_support_x86_64.h index a4446d3345..51befbe7b8 100644 --- a/runtime/arch/x86_64/asm_support_x86_64.h +++ b/runtime/arch/x86_64/asm_support_x86_64.h @@ -23,5 +23,7 @@ #define FRAME_SIZE_SAVE_REFS_ONLY (64 + 4*8) #define FRAME_SIZE_SAVE_REFS_AND_ARGS (112 + 12*8) #define FRAME_SIZE_SAVE_EVERYTHING (144 + 16*8) +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING +#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING #endif // ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_H_ diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S index 0a9199e7e9..73e86100f6 100644 --- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S +++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S @@ -272,7 +272,7 @@ END_MACRO * Runtime::CreateCalleeSaveMethod(kSaveEverything) * when R14 and R15 are already saved. */ -MACRO0(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED) +MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) #if defined(__APPLE__) int3 int3 @@ -316,7 +316,7 @@ MACRO0(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED) movq %xmm14, 120(%rsp) movq %xmm15, 128(%rsp) // Push ArtMethod* for save everything frame method. - pushq RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET(%r10) + pushq \runtime_method_offset(%r10) CFI_ADJUST_CFA_OFFSET(8) // Store rsp as the top quick frame. movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET @@ -334,18 +334,18 @@ END_MACRO * Runtime::CreateCalleeSaveMethod(kSaveEverything) * when R15 is already saved. */ -MACRO0(SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED) +MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) PUSH r14 - SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED + SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED \runtime_method_offset END_MACRO /* * Macro that sets up the callee save frame to conform with * Runtime::CreateCalleeSaveMethod(kSaveEverything) */ -MACRO0(SETUP_SAVE_EVERYTHING_FRAME) +MACRO1(SETUP_SAVE_EVERYTHING_FRAME, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) PUSH r15 - SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED + SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED \runtime_method_offset END_MACRO MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_FRPS) @@ -951,9 +951,9 @@ MACRO3(THREE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) END_MACRO // Macro for string and type resolution and initialization. -MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name) +MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) DEFINE_FUNCTION VAR(c_name) - SETUP_SAVE_EVERYTHING_FRAME // save everything for GC + SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for GC // Outgoing argument set up movl %eax, %edi // pass string index movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() @@ -970,6 +970,10 @@ MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name) END_FUNCTION VAR(c_name) END_MACRO +MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name) + ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET +END_MACRO + MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER) testq %rax, %rax // rax == 0 ? jz 1f // if rax == 0 goto 1 @@ -1290,8 +1294,8 @@ DEFINE_FUNCTION art_quick_alloc_object_initialized_region_tlab ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedRegionTLAB END_FUNCTION art_quick_alloc_object_initialized_region_tlab -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode -ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode +ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode @@ -1575,7 +1579,7 @@ DEFINE_FUNCTION art_quick_memcpy END_FUNCTION art_quick_memcpy DEFINE_FUNCTION art_quick_test_suspend - SETUP_SAVE_EVERYTHING_FRAME // save everything for GC + SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save everything for GC // Outgoing argument set up movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current() call SYMBOL(artTestSuspendFromCode) // (Thread*) diff --git a/runtime/arch/x86_64/quick_method_frame_info_x86_64.h b/runtime/arch/x86_64/quick_method_frame_info_x86_64.h index 425d616e76..ebf976ef71 100644 --- a/runtime/arch/x86_64/quick_method_frame_info_x86_64.h +++ b/runtime/arch/x86_64/quick_method_frame_info_x86_64.h @@ -56,24 +56,28 @@ static constexpr uint32_t kX86_64CalleeSaveFpEverythingSpills = (1 << art::x86_64::XMM10) | (1 << art::x86_64::XMM11); constexpr uint32_t X86_64CalleeSaveCoreSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kX86_64CalleeSaveAlwaysSpills | kX86_64CalleeSaveRefSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kX86_64CalleeSaveArgSpills : 0) | (type == CalleeSaveType::kSaveEverything ? kX86_64CalleeSaveEverythingSpills : 0); } constexpr uint32_t X86_64CalleeSaveFpSpills(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return kX86_64CalleeSaveFpSpills | (type == CalleeSaveType::kSaveRefsAndArgs ? kX86_64CalleeSaveFpArgSpills : 0) | (type == CalleeSaveType::kSaveEverything ? kX86_64CalleeSaveFpEverythingSpills : 0); } constexpr uint32_t X86_64CalleeSaveFrameSize(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return RoundUp((POPCOUNT(X86_64CalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(X86_64CalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kX86_64PointerSize), kStackAlignment); } constexpr QuickMethodFrameInfo X86_64CalleeSaveMethodFrameInfo(CalleeSaveType type) { + type = GetCanonicalCalleeSaveType(type); return QuickMethodFrameInfo(X86_64CalleeSaveFrameSize(type), X86_64CalleeSaveCoreSpills(type), X86_64CalleeSaveFpSpills(type)); diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h index fad927804e..50e91447a9 100644 --- a/runtime/art_method-inl.h +++ b/runtime/art_method-inl.h @@ -285,6 +285,10 @@ inline const char* ArtMethod::GetName() { return "<runtime internal callee-save reference and argument registers method>"; } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything)) { return "<runtime internal save-every-register method>"; + } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit)) { + return "<runtime internal save-every-register method for clinit>"; + } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck)) { + return "<runtime internal save-every-register method for suspend check>"; } else { return "<unknown runtime internal method>"; } diff --git a/runtime/base/callee_save_type.h b/runtime/base/callee_save_type.h index 501b296d4f..e9cd63c3a0 100644 --- a/runtime/base/callee_save_type.h +++ b/runtime/base/callee_save_type.h @@ -28,10 +28,20 @@ enum class CalleeSaveType : uint32_t { kSaveRefsOnly, // Only those callee-save registers that can hold references. kSaveRefsAndArgs, // References (see above) and arguments (usually caller-save registers). kSaveEverything, // All registers, including both callee-save and caller-save. + kSaveEverythingForClinit, // Special kSaveEverything for clinit. + kSaveEverythingForSuspendCheck, // Special kSaveEverything for suspend check. kLastCalleeSaveType // Value used for iteration. }; std::ostream& operator<<(std::ostream& os, const CalleeSaveType& rhs); +static inline constexpr CalleeSaveType GetCanonicalCalleeSaveType(CalleeSaveType type) { + if (type == CalleeSaveType::kSaveEverythingForClinit || + type == CalleeSaveType::kSaveEverythingForSuspendCheck) { + return CalleeSaveType::kSaveEverything; + } + return type; +} + } // namespace art #endif // ART_RUNTIME_BASE_CALLEE_SAVE_TYPE_H_ diff --git a/runtime/base/unix_file/fd_file.cc b/runtime/base/unix_file/fd_file.cc index 0c73ce7ef2..eb8ced00a8 100644 --- a/runtime/base/unix_file/fd_file.cc +++ b/runtime/base/unix_file/fd_file.cc @@ -438,4 +438,30 @@ bool FdFile::ResetOffset() { return true; } +int FdFile::Compare(FdFile* other) { + int64_t length = GetLength(); + int64_t length2 = other->GetLength(); + if (length != length2) { + return length < length2 ? -1 : 1; + } + static const size_t kBufferSize = 4096; + std::unique_ptr<uint8_t[]> buffer1(new uint8_t[kBufferSize]); + std::unique_ptr<uint8_t[]> buffer2(new uint8_t[kBufferSize]); + while (length > 0) { + size_t len = std::min(kBufferSize, static_cast<size_t>(length)); + if (!ReadFully(&buffer1[0], len)) { + return -1; + } + if (!other->ReadFully(&buffer2[0], len)) { + return 1; + } + int result = memcmp(&buffer1[0], &buffer2[0], len); + if (result != 0) { + return result; + } + length -= len; + } + return 0; +} + } // namespace unix_file diff --git a/runtime/base/unix_file/fd_file.h b/runtime/base/unix_file/fd_file.h index e07c3fd800..91b08bcb4d 100644 --- a/runtime/base/unix_file/fd_file.h +++ b/runtime/base/unix_file/fd_file.h @@ -145,6 +145,11 @@ class FdFile : public RandomAccessFile { // WARNING: Only use this when you know what you're doing! void MarkUnchecked(); + // Compare against another file. Returns 0 if the files are equivalent, otherwise returns -1 or 1 + // depending on if the lenghts are different. If the lengths are the same, the function returns + // the difference of the first byte that differs. + int Compare(FdFile* other); + protected: // If the guard state indicates checking (!=kNoCheck), go to the target state "target". Print the // given warning if the current state is or exceeds warn_threshold. diff --git a/runtime/base/unix_file/fd_file_test.cc b/runtime/base/unix_file/fd_file_test.cc index 6aef348433..8b1a115a26 100644 --- a/runtime/base/unix_file/fd_file_test.cc +++ b/runtime/base/unix_file/fd_file_test.cc @@ -220,4 +220,58 @@ TEST_F(FdFileTest, EraseWithPathUnlinks) { EXPECT_FALSE(art::OS::FileExists(filename.c_str())) << filename; } +TEST_F(FdFileTest, Compare) { + std::vector<uint8_t> buffer; + constexpr int64_t length = 17 * art::KB; + for (size_t i = 0; i < length; ++i) { + buffer.push_back(static_cast<uint8_t>(i)); + } + + auto reset_compare = [&](art::ScratchFile& a, art::ScratchFile& b) { + a.GetFile()->ResetOffset(); + b.GetFile()->ResetOffset(); + return a.GetFile()->Compare(b.GetFile()); + }; + + art::ScratchFile tmp; + EXPECT_TRUE(tmp.GetFile()->WriteFully(&buffer[0], length)); + EXPECT_EQ(tmp.GetFile()->GetLength(), length); + + art::ScratchFile tmp2; + EXPECT_TRUE(tmp2.GetFile()->WriteFully(&buffer[0], length)); + EXPECT_EQ(tmp2.GetFile()->GetLength(), length); + + // Basic equality check. + tmp.GetFile()->ResetOffset(); + tmp2.GetFile()->ResetOffset(); + // Files should be the same. + EXPECT_EQ(reset_compare(tmp, tmp2), 0); + + // Change a byte near the start. + ++buffer[2]; + art::ScratchFile tmp3; + EXPECT_TRUE(tmp3.GetFile()->WriteFully(&buffer[0], length)); + --buffer[2]; + EXPECT_NE(reset_compare(tmp, tmp3), 0); + + // Change a byte near the middle. + ++buffer[length / 2]; + art::ScratchFile tmp4; + EXPECT_TRUE(tmp4.GetFile()->WriteFully(&buffer[0], length)); + --buffer[length / 2]; + EXPECT_NE(reset_compare(tmp, tmp4), 0); + + // Change a byte near the end. + ++buffer[length - 5]; + art::ScratchFile tmp5; + EXPECT_TRUE(tmp5.GetFile()->WriteFully(&buffer[0], length)); + --buffer[length - 5]; + EXPECT_NE(reset_compare(tmp, tmp5), 0); + + // Reference check + art::ScratchFile tmp6; + EXPECT_TRUE(tmp6.GetFile()->WriteFully(&buffer[0], length)); + EXPECT_EQ(reset_compare(tmp, tmp6), 0); +} + } // namespace unix_file diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index 3ac87c5137..9756f5787f 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -3694,7 +3694,8 @@ bool ClassLinker::IsDexFileRegistered(Thread* self, const DexFile& dex_file) { ObjPtr<mirror::DexCache> ClassLinker::FindDexCache(Thread* self, const DexFile& dex_file) { ReaderMutexLock mu(self, *Locks::dex_lock_); - ObjPtr<mirror::DexCache> dex_cache = DecodeDexCache(self, FindDexCacheDataLocked(dex_file)); + DexCacheData dex_cache_data = FindDexCacheDataLocked(dex_file); + ObjPtr<mirror::DexCache> dex_cache = DecodeDexCache(self, dex_cache_data); if (dex_cache != nullptr) { return dex_cache; } @@ -3704,7 +3705,8 @@ ObjPtr<mirror::DexCache> ClassLinker::FindDexCache(Thread* self, const DexFile& LOG(FATAL_WITHOUT_ABORT) << "Registered dex file " << data.dex_file->GetLocation(); } } - LOG(FATAL) << "Failed to find DexCache for DexFile " << dex_file.GetLocation(); + LOG(FATAL) << "Failed to find DexCache for DexFile " << dex_file.GetLocation() + << " " << &dex_file << " " << dex_cache_data.dex_file; UNREACHABLE(); } @@ -4280,13 +4282,7 @@ verifier::FailureKind ClassLinker::VerifyClass( std::string error_msg; verifier::FailureKind verifier_failure = verifier::FailureKind::kNoFailure; if (!preverified) { - Runtime* runtime = Runtime::Current(); - verifier_failure = verifier::MethodVerifier::VerifyClass(self, - klass.Get(), - runtime->GetCompilerCallbacks(), - runtime->IsAotCompiler(), - log_level, - &error_msg); + verifier_failure = PerformClassVerification(self, klass, log_level, &error_msg); } // Verification is done, grab the lock again. @@ -4354,6 +4350,19 @@ verifier::FailureKind ClassLinker::VerifyClass( return verifier_failure; } +verifier::FailureKind ClassLinker::PerformClassVerification(Thread* self, + Handle<mirror::Class> klass, + verifier::HardFailLogMode log_level, + std::string* error_msg) { + Runtime* const runtime = Runtime::Current(); + return verifier::MethodVerifier::VerifyClass(self, + klass.Get(), + runtime->GetCompilerCallbacks(), + runtime->IsAotCompiler(), + log_level, + error_msg); +} + bool ClassLinker::VerifyClassUsingOatFile(const DexFile& dex_file, ObjPtr<mirror::Class> klass, mirror::Class::Status& oat_file_class_status) { @@ -7119,7 +7128,7 @@ void ClassLinker::LinkInterfaceMethodsHelper::ReallocMethods() { method_alignment_); const size_t old_methods_ptr_size = (old_methods != nullptr) ? old_size : 0; auto* methods = reinterpret_cast<LengthPrefixedArray<ArtMethod>*>( - Runtime::Current()->GetLinearAlloc()->Realloc( + class_linker_->GetAllocatorForClassLoader(klass_->GetClassLoader())->Realloc( self_, old_methods, old_methods_ptr_size, new_size)); CHECK(methods != nullptr); // Native allocation failure aborts. diff --git a/runtime/class_linker.h b/runtime/class_linker.h index bf14aebb52..2fbbe79e47 100644 --- a/runtime/class_linker.h +++ b/runtime/class_linker.h @@ -710,6 +710,12 @@ class ClassLinker { REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Locks::dex_lock_); + virtual verifier::FailureKind PerformClassVerification(Thread* self, + Handle<mirror::Class> klass, + verifier::HardFailLogMode log_level, + std::string* error_msg) + REQUIRES_SHARED(Locks::mutator_lock_); + private: class LinkInterfaceMethodsHelper; diff --git a/runtime/class_loader_context.cc b/runtime/class_loader_context.cc index b50aec0b63..e7051b35d8 100644 --- a/runtime/class_loader_context.cc +++ b/runtime/class_loader_context.cc @@ -68,6 +68,10 @@ ClassLoaderContext::~ClassLoaderContext() { } } +std::unique_ptr<ClassLoaderContext> ClassLoaderContext::Default() { + return Create(""); +} + std::unique_ptr<ClassLoaderContext> ClassLoaderContext::Create(const std::string& spec) { std::unique_ptr<ClassLoaderContext> result(new ClassLoaderContext()); if (result->Parse(spec)) { diff --git a/runtime/class_loader_context.h b/runtime/class_loader_context.h index 85299d41c0..9afa880da4 100644 --- a/runtime/class_loader_context.h +++ b/runtime/class_loader_context.h @@ -34,9 +34,6 @@ class OatFile; // Utility class which holds the class loader context used during compilation/verification. class ClassLoaderContext { public: - // Creates an empty context (with no class loaders). - ClassLoaderContext(); - ~ClassLoaderContext(); // Opens requested class path files and appends them to ClassLoaderInfo::opened_dex_files. @@ -126,6 +123,10 @@ class ClassLoaderContext { static std::unique_ptr<ClassLoaderContext> CreateContextForClassLoader(jobject class_loader, jobjectArray dex_elements); + // Returns the default class loader context to be used when none is specified. + // This will return a context with a single and empty PathClassLoader. + static std::unique_ptr<ClassLoaderContext> Default(); + private: enum ClassLoaderType { kInvalidClassLoader = 0, @@ -151,6 +152,9 @@ class ClassLoaderContext { explicit ClassLoaderInfo(ClassLoaderType cl_type) : type(cl_type) {} }; + // Creates an empty context (with no class loaders). + ClassLoaderContext(); + // Constructs an empty context. // `owns_the_dex_files` specifies whether or not the context will own the opened dex files // present in the class loader chain. If `owns_the_dex_files` is true then OpenDexFiles cannot diff --git a/runtime/compiler_callbacks.h b/runtime/compiler_callbacks.h index 806653a265..c51bb5e176 100644 --- a/runtime/compiler_callbacks.h +++ b/runtime/compiler_callbacks.h @@ -22,6 +22,8 @@ namespace art { +class CompilerDriver; + namespace verifier { class MethodVerifier; @@ -49,6 +51,13 @@ class CompilerCallbacks { virtual verifier::VerifierDeps* GetVerifierDeps() const = 0; virtual void SetVerifierDeps(verifier::VerifierDeps* deps ATTRIBUTE_UNUSED) {} + virtual bool CanAssumeVerified(ClassReference ref ATTRIBUTE_UNUSED) { + return false; + } + + virtual void SetDoesClassUnloading(bool does_class_unloading ATTRIBUTE_UNUSED, + CompilerDriver* compiler_driver ATTRIBUTE_UNUSED) {} + bool IsBootImage() { return mode_ == CallbackMode::kCompileBootImage; } diff --git a/runtime/dex_file_verifier.cc b/runtime/dex_file_verifier.cc index c5c4eda98f..8fdd4706e4 100644 --- a/runtime/dex_file_verifier.cc +++ b/runtime/dex_file_verifier.cc @@ -363,7 +363,7 @@ bool DexFileVerifier::CheckHeader() { // Check file size from the header. uint32_t expected_size = header_->file_size_; if (size_ != expected_size) { - ErrorStringPrintf("Bad file size (%zd, expected %ud)", size_, expected_size); + ErrorStringPrintf("Bad file size (%zd, expected %u)", size_, expected_size); return false; } diff --git a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc index a2a6e085c2..5355267b07 100644 --- a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc @@ -65,8 +65,8 @@ extern "C" mirror::Class* artInitializeStaticStorageFromCode(uint32_t type_idx, // A class may be accessing another class' fields when it doesn't have access, as access has been // given by inheritance. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, - CalleeSaveType::kSaveEverything); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod( + self, CalleeSaveType::kSaveEverythingForClinit); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, true, false); @@ -80,8 +80,8 @@ extern "C" mirror::Class* artInitializeTypeFromCode(uint32_t type_idx, Thread* s REQUIRES_SHARED(Locks::mutator_lock_) { // Called when method->dex_cache_resolved_types_[] misses. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, - CalleeSaveType::kSaveEverything); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod( + self, CalleeSaveType::kSaveEverythingForClinit); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, false, false); diff --git a/runtime/entrypoints/quick/quick_field_entrypoints.cc b/runtime/entrypoints/quick/quick_field_entrypoints.cc index 726bddd334..b298db674c 100644 --- a/runtime/entrypoints/quick/quick_field_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_field_entrypoints.cc @@ -301,6 +301,7 @@ extern "C" mirror::Object* artReadBarrierMark(mirror::Object* obj) { extern "C" mirror::Object* artReadBarrierSlow(mirror::Object* ref ATTRIBUTE_UNUSED, mirror::Object* obj, uint32_t offset) { + // Used only in connection with non-volatile loads. DCHECK(kEmitCompilerReadBarrier); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(obj) + offset; mirror::HeapReference<mirror::Object>* ref_addr = @@ -308,9 +309,10 @@ extern "C" mirror::Object* artReadBarrierSlow(mirror::Object* ref ATTRIBUTE_UNUS constexpr ReadBarrierOption kReadBarrierOption = kUseReadBarrier ? kWithReadBarrier : kWithoutReadBarrier; mirror::Object* result = - ReadBarrier::Barrier<mirror::Object, kReadBarrierOption>(obj, - MemberOffset(offset), - ref_addr); + ReadBarrier::Barrier<mirror::Object, /* kIsVolatile */ false, kReadBarrierOption>( + obj, + MemberOffset(offset), + ref_addr); return result; } diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc index 7e08b7ace0..b692618740 100644 --- a/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc +++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc @@ -90,7 +90,18 @@ TEST_F(QuickTrampolineEntrypointsTest, FrameSize) { GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveRefsOnly)); \ CheckFrameSize(isa, \ CalleeSaveType::kSaveAllCalleeSaves, \ - GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveAllCalleeSaves)) + GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveAllCalleeSaves)); \ + CheckFrameSize(isa, \ + CalleeSaveType::kSaveEverything, \ + GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveEverything)); \ + CheckFrameSize(isa, \ + CalleeSaveType::kSaveEverythingForClinit, \ + GetCalleeSaveFrameSize(isa, \ + CalleeSaveType::kSaveEverythingForClinit)); \ + CheckFrameSize(isa, \ + CalleeSaveType::kSaveEverythingForSuspendCheck, \ + GetCalleeSaveFrameSize( \ + isa, CalleeSaveType::kSaveEverythingForSuspendCheck)) CHECK_FRAME_SIZE(kArm); CHECK_FRAME_SIZE(kArm64); @@ -123,6 +134,13 @@ TEST_F(QuickTrampolineEntrypointsTest, ReturnPC) { GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveRefsOnly)); CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveAllCalleeSaves, GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveAllCalleeSaves)); + CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveEverything, + GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveEverything)); + CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveEverythingForClinit, + GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveEverythingForClinit)); + CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveEverythingForSuspendCheck, + GetCalleeSaveReturnPcOffset(kRuntimeISA, + CalleeSaveType::kSaveEverythingForSuspendCheck)); } } // namespace art diff --git a/runtime/fault_handler.cc b/runtime/fault_handler.cc index fd0cd5f0b2..7d01af02a3 100644 --- a/runtime/fault_handler.cc +++ b/runtime/fault_handler.cc @@ -79,8 +79,7 @@ static mirror::Class* SafeGetDeclaringClass(ArtMethod* method) static mirror::Class* SafeGetClass(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) { char* obj_cls = reinterpret_cast<char*>(obj) + mirror::Object::ClassOffset().SizeValue(); - mirror::HeapReference<mirror::Class> cls = - mirror::HeapReference<mirror::Class>::FromMirrorPtr(nullptr); + mirror::HeapReference<mirror::Class> cls; ssize_t rc = SafeCopy(&cls, obj_cls, sizeof(cls)); CHECK_NE(-1, rc); diff --git a/runtime/gc/accounting/mod_union_table.cc b/runtime/gc/accounting/mod_union_table.cc index 290199579b..1b3d0dadae 100644 --- a/runtime/gc/accounting/mod_union_table.cc +++ b/runtime/gc/accounting/mod_union_table.cc @@ -115,8 +115,8 @@ class ModUnionUpdateObjectReferencesVisitor { } private: - template<bool kPoisonReferences> - void MarkReference(mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) const + template<typename CompressedReferenceType> + void MarkReference(CompressedReferenceType* obj_ptr) const REQUIRES_SHARED(Locks::mutator_lock_) { // Only add the reference if it is non null and fits our criteria. mirror::Object* ref = obj_ptr->AsMirrorPtr(); diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc index 92b4360123..280c0b1d69 100644 --- a/runtime/gc/accounting/space_bitmap.cc +++ b/runtime/gc/accounting/space_bitmap.cc @@ -48,16 +48,21 @@ SpaceBitmap<kAlignment>* SpaceBitmap<kAlignment>::CreateFromMemMap( CHECK(mem_map != nullptr); uintptr_t* bitmap_begin = reinterpret_cast<uintptr_t*>(mem_map->Begin()); const size_t bitmap_size = ComputeBitmapSize(heap_capacity); - return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin); + return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin, heap_capacity); } template<size_t kAlignment> -SpaceBitmap<kAlignment>::SpaceBitmap(const std::string& name, MemMap* mem_map, uintptr_t* bitmap_begin, - size_t bitmap_size, const void* heap_begin) +SpaceBitmap<kAlignment>::SpaceBitmap(const std::string& name, + MemMap* mem_map, + uintptr_t* bitmap_begin, + size_t bitmap_size, + const void* heap_begin, + size_t heap_capacity) : mem_map_(mem_map), bitmap_begin_(reinterpret_cast<Atomic<uintptr_t>*>(bitmap_begin)), bitmap_size_(bitmap_size), heap_begin_(reinterpret_cast<uintptr_t>(heap_begin)), + heap_limit_(reinterpret_cast<uintptr_t>(heap_begin) + heap_capacity), name_(name) { CHECK(bitmap_begin_ != nullptr); CHECK_NE(bitmap_size, 0U); @@ -89,6 +94,7 @@ void SpaceBitmap<kAlignment>::SetHeapLimit(uintptr_t new_end) { if (new_size < bitmap_size_) { bitmap_size_ = new_size; } + heap_limit_ = new_end; // Not sure if doing this trim is necessary, since nothing past the end of the heap capacity // should be marked. } diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h index 2fe6394c0f..b49e0b7f89 100644 --- a/runtime/gc/accounting/space_bitmap.h +++ b/runtime/gc/accounting/space_bitmap.h @@ -163,6 +163,7 @@ class SpaceBitmap { void SetHeapSize(size_t bytes) { // TODO: Un-map the end of the mem map. + heap_limit_ = heap_begin_ + bytes; bitmap_size_ = OffsetToIndex(bytes) * sizeof(intptr_t); CHECK_EQ(HeapSize(), bytes); } @@ -173,7 +174,7 @@ class SpaceBitmap { // The maximum address which the bitmap can span. (HeapBegin() <= object < HeapLimit()). uint64_t HeapLimit() const { - return static_cast<uint64_t>(HeapBegin()) + HeapSize(); + return heap_limit_; } // Set the max address which can covered by the bitmap. @@ -196,8 +197,12 @@ class SpaceBitmap { private: // TODO: heap_end_ is initialized so that the heap bitmap is empty, this doesn't require the -1, // however, we document that this is expected on heap_end_ - SpaceBitmap(const std::string& name, MemMap* mem_map, uintptr_t* bitmap_begin, size_t bitmap_size, - const void* heap_begin); + SpaceBitmap(const std::string& name, + MemMap* mem_map, + uintptr_t* bitmap_begin, + size_t bitmap_size, + const void* heap_begin, + size_t heap_capacity); template<bool kSetBit> bool Modify(const mirror::Object* obj); @@ -211,10 +216,13 @@ class SpaceBitmap { // Size of this bitmap. size_t bitmap_size_; - // The base address of the heap, which corresponds to the word containing the first bit in the - // bitmap. + // The start address of the memory covered by the bitmap, which corresponds to the word + // containing the first bit in the bitmap. const uintptr_t heap_begin_; + // The end address of the memory covered by the bitmap. This may not be on a word boundary. + uintptr_t heap_limit_; + // Name of this bitmap. std::string name_; }; diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index 9d672b1d34..52b355dedd 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -2431,24 +2431,31 @@ mirror::Object* ConcurrentCopying::IsMarked(mirror::Object* from_ref) { // Non-immune non-moving space. Use the mark bitmap. accounting::ContinuousSpaceBitmap* mark_bitmap = heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref); - accounting::LargeObjectBitmap* los_bitmap = - heap_mark_bitmap_->GetLargeObjectBitmap(from_ref); - CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; bool is_los = mark_bitmap == nullptr; if (!is_los && mark_bitmap->Test(from_ref)) { // Already marked. to_ref = from_ref; - } else if (is_los && los_bitmap->Test(from_ref)) { - // Already marked in LOS. - to_ref = from_ref; } else { - // Not marked. - if (IsOnAllocStack(from_ref)) { - // If on the allocation stack, it's considered marked. + accounting::LargeObjectBitmap* los_bitmap = + heap_mark_bitmap_->GetLargeObjectBitmap(from_ref); + // We may not have a large object space for dex2oat, don't assume it exists. + if (los_bitmap == nullptr) { + CHECK(heap_->GetLargeObjectsSpace() == nullptr) + << "LOS bitmap covers the entire address range " << from_ref + << " " << heap_->DumpSpaces(); + } + if (los_bitmap != nullptr && is_los && los_bitmap->Test(from_ref)) { + // Already marked in LOS. to_ref = from_ref; } else { // Not marked. - to_ref = nullptr; + if (IsOnAllocStack(from_ref)) { + // If on the allocation stack, it's considered marked. + to_ref = from_ref; + } else { + // Not marked. + to_ref = nullptr; + } } } } @@ -2457,6 +2464,7 @@ mirror::Object* ConcurrentCopying::IsMarked(mirror::Object* from_ref) { } bool ConcurrentCopying::IsOnAllocStack(mirror::Object* ref) { + // TODO: Explain why this is here. What release operation does it pair with? QuasiAtomic::ThreadFenceAcquire(); accounting::ObjectStack* alloc_stack = GetAllocationStack(); return alloc_stack->Contains(ref); @@ -2617,9 +2625,8 @@ bool ConcurrentCopying::IsNullOrMarkedHeapReference(mirror::HeapReference<mirror } } while (!field->CasWeakRelaxed(from_ref, to_ref)); } else { - QuasiAtomic::ThreadFenceRelease(); - field->Assign(to_ref); - QuasiAtomic::ThreadFenceSequentiallyConsistent(); + // TODO: Why is this seq_cst when the above is relaxed? Document memory ordering. + field->Assign</* kIsVolatile */ true>(to_ref); } } return true; diff --git a/runtime/gc/collector/garbage_collector.h b/runtime/gc/collector/garbage_collector.h index dec206be30..f722e8d855 100644 --- a/runtime/gc/collector/garbage_collector.h +++ b/runtime/gc/collector/garbage_collector.h @@ -113,7 +113,7 @@ class GarbageCollector : public RootVisitor, public IsMarkedVisitor, public Mark virtual mirror::Object* IsMarked(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) = 0; // Returns true if the given heap reference is null or is already marked. If it's already marked, - // update the reference (uses a CAS if do_atomic_update is true. Otherwise, returns false. + // update the reference (uses a CAS if do_atomic_update is true). Otherwise, returns false. virtual bool IsNullOrMarkedHeapReference(mirror::HeapReference<mirror::Object>* obj, bool do_atomic_update) REQUIRES_SHARED(Locks::mutator_lock_) = 0; diff --git a/runtime/gc/collector/semi_space-inl.h b/runtime/gc/collector/semi_space-inl.h index 78fb2d24ae..7db5d2ca20 100644 --- a/runtime/gc/collector/semi_space-inl.h +++ b/runtime/gc/collector/semi_space-inl.h @@ -38,9 +38,8 @@ inline mirror::Object* SemiSpace::GetForwardingAddressInFromSpace(mirror::Object // Used to mark and copy objects. Any newly-marked objects who are in the from space Get moved to // the to-space and have their forward address updated. Objects which have been newly marked are // pushed on the mark stack. -template<bool kPoisonReferences> -inline void SemiSpace::MarkObject( - mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) { +template<typename CompressedReferenceType> +inline void SemiSpace::MarkObject(CompressedReferenceType* obj_ptr) { mirror::Object* obj = obj_ptr->AsMirrorPtr(); if (obj == nullptr) { return; @@ -73,9 +72,8 @@ inline void SemiSpace::MarkObject( } } -template<bool kPoisonReferences> -inline void SemiSpace::MarkObjectIfNotInToSpace( - mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) { +template<typename CompressedReferenceType> +inline void SemiSpace::MarkObjectIfNotInToSpace(CompressedReferenceType* obj_ptr) { if (!to_space_->HasAddress(obj_ptr->AsMirrorPtr())) { MarkObject(obj_ptr); } diff --git a/runtime/gc/collector/semi_space.h b/runtime/gc/collector/semi_space.h index fd52da3947..6d4d789a72 100644 --- a/runtime/gc/collector/semi_space.h +++ b/runtime/gc/collector/semi_space.h @@ -97,13 +97,13 @@ class SemiSpace : public GarbageCollector { // Find the default mark bitmap. void FindDefaultMarkBitmap(); - // Updates obj_ptr if the object has moved. - template<bool kPoisonReferences> - void MarkObject(mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) + // Updates obj_ptr if the object has moved. Takes either an ObjectReference or a HeapReference. + template<typename CompressedReferenceType> + void MarkObject(CompressedReferenceType* obj_ptr) REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - template<bool kPoisonReferences> - void MarkObjectIfNotInToSpace(mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) + template<typename CompressedReferenceType> + void MarkObjectIfNotInToSpace(CompressedReferenceType* obj_ptr) REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); virtual mirror::Object* MarkObject(mirror::Object* root) OVERRIDE diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index 148438296f..14e017abd9 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -725,6 +725,10 @@ class ImageSpaceLoader { image_header->GetImageMethod(ImageHeader::kSaveRefsAndArgsMethod)); CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything), image_header->GetImageMethod(ImageHeader::kSaveEverythingMethod)); + CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit), + image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForClinit)); + CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck), + image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForSuspendCheck)); } else if (!runtime->HasResolutionMethod()) { runtime->SetInstructionSet(space->oat_file_non_owned_->GetOatHeader().GetInstructionSet()); runtime->SetResolutionMethod(image_header->GetImageMethod(ImageHeader::kResolutionMethod)); @@ -743,6 +747,12 @@ class ImageSpaceLoader { runtime->SetCalleeSaveMethod( image_header->GetImageMethod(ImageHeader::kSaveEverythingMethod), CalleeSaveType::kSaveEverything); + runtime->SetCalleeSaveMethod( + image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForClinit), + CalleeSaveType::kSaveEverythingForClinit); + runtime->SetCalleeSaveMethod( + image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForSuspendCheck), + CalleeSaveType::kSaveEverythingForSuspendCheck); } VLOG(image) << "ImageSpace::Init exiting " << *space.get(); diff --git a/runtime/gc/space/large_object_space_test.cc b/runtime/gc/space/large_object_space_test.cc index 79b775a510..9baa016dcd 100644 --- a/runtime/gc/space/large_object_space_test.cc +++ b/runtime/gc/space/large_object_space_test.cc @@ -38,12 +38,19 @@ void LargeObjectSpaceTest::LargeObjectTest() { Thread* const self = Thread::Current(); for (size_t i = 0; i < 2; ++i) { LargeObjectSpace* los = nullptr; + const size_t capacity = 128 * MB; if (i == 0) { los = space::LargeObjectMapSpace::Create("large object space"); } else { - los = space::FreeListSpace::Create("large object space", nullptr, 128 * MB); + los = space::FreeListSpace::Create("large object space", nullptr, capacity); } + // Make sure the bitmap is not empty and actually covers at least how much we expect. + CHECK_LT(static_cast<uintptr_t>(los->GetLiveBitmap()->HeapBegin()), + static_cast<uintptr_t>(los->GetLiveBitmap()->HeapLimit())); + CHECK_LE(static_cast<uintptr_t>(los->GetLiveBitmap()->HeapBegin() + capacity), + static_cast<uintptr_t>(los->GetLiveBitmap()->HeapLimit())); + static const size_t num_allocations = 64; static const size_t max_allocation_size = 0x100000; std::vector<std::pair<mirror::Object*, size_t>> requests; diff --git a/runtime/gc/space/malloc_space.cc b/runtime/gc/space/malloc_space.cc index c2a8de3aec..c99412785e 100644 --- a/runtime/gc/space/malloc_space.cc +++ b/runtime/gc/space/malloc_space.cc @@ -191,14 +191,10 @@ ZygoteSpace* MallocSpace::CreateZygoteSpace(const char* alloc_space_name, bool l VLOG(heap) << "Size " << GetMemMap()->Size(); VLOG(heap) << "GrowthLimit " << PrettySize(growth_limit); VLOG(heap) << "Capacity " << PrettySize(capacity); - // Remap the tail. Pass MAP_PRIVATE since we don't want to share the same ashmem as the zygote - // space. + // Remap the tail. std::string error_msg; - std::unique_ptr<MemMap> mem_map(GetMemMap()->RemapAtEnd(End(), - alloc_space_name, - PROT_READ | PROT_WRITE, - MAP_PRIVATE, - &error_msg)); + std::unique_ptr<MemMap> mem_map(GetMemMap()->RemapAtEnd(End(), alloc_space_name, + PROT_READ | PROT_WRITE, &error_msg)); CHECK(mem_map.get() != nullptr) << error_msg; void* allocator = CreateAllocator(End(), starting_size_, initial_size_, capacity, low_memory_mode); diff --git a/runtime/generated/asm_support_gen.h b/runtime/generated/asm_support_gen.h index 11b3abb6ed..314c45e117 100644 --- a/runtime/generated/asm_support_gen.h +++ b/runtime/generated/asm_support_gen.h @@ -34,6 +34,10 @@ DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET), (stat DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveRefsAndArgs)))) #define RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 0x18 DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveEverything)))) +#define RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET 0x20 +DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveEverythingForClinit)))) +#define RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET 0x28 +DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveEverythingForSuspendCheck)))) #define THREAD_FLAGS_OFFSET 0 DEFINE_CHECK_EQ(static_cast<int32_t>(THREAD_FLAGS_OFFSET), (static_cast<int32_t>(art::Thread:: ThreadFlagsOffset<art::kRuntimePointerSize>().Int32Value()))) #define THREAD_ID_OFFSET 12 diff --git a/runtime/image.h b/runtime/image.h index 6c76f4915e..7bb796c5a1 100644 --- a/runtime/image.h +++ b/runtime/image.h @@ -183,6 +183,8 @@ class PACKED(4) ImageHeader { kSaveRefsOnlyMethod, kSaveRefsAndArgsMethod, kSaveEverythingMethod, + kSaveEverythingMethodForClinit, + kSaveEverythingMethodForSuspendCheck, kImageMethodsCount, // Number of elements in enum. }; diff --git a/runtime/interpreter/interpreter_common.h b/runtime/interpreter/interpreter_common.h index b228e28a22..5b942f2e73 100644 --- a/runtime/interpreter/interpreter_common.h +++ b/runtime/interpreter/interpreter_common.h @@ -141,11 +141,8 @@ static inline bool DoFastInvoke(Thread* self, return false; } else { jit::Jit* jit = Runtime::Current()->GetJit(); - if (jit != nullptr) { - if (type == kVirtual) { - jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method); - } - jit->AddSamples(self, sf_method, 1, /*with_backedges*/false); + if (jit != nullptr && type == kVirtual) { + jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method); } if (called_method->IsIntrinsic()) { if (MterpHandleIntrinsic(&shadow_frame, called_method, inst, inst_data, @@ -182,11 +179,8 @@ static inline bool DoInvoke(Thread* self, return false; } else { jit::Jit* jit = Runtime::Current()->GetJit(); - if (jit != nullptr) { - if (type == kVirtual || type == kInterface) { - jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method); - } - jit->AddSamples(self, sf_method, 1, /*with_backedges*/false); + if (jit != nullptr && (type == kVirtual || type == kInterface)) { + jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method); } // TODO: Remove the InvokeVirtualOrInterface instrumentation, as it was only used by the JIT. if (type == kVirtual || type == kInterface) { diff --git a/runtime/interpreter/mterp/mterp.cc b/runtime/interpreter/mterp/mterp.cc index 5955b9001a..88254a8cdc 100644 --- a/runtime/interpreter/mterp/mterp.cc +++ b/runtime/interpreter/mterp/mterp.cc @@ -280,7 +280,6 @@ extern "C" size_t MterpInvokeVirtualQuick(Thread* self, if (jit != nullptr) { jit->InvokeVirtualOrInterface( receiver, shadow_frame->GetMethod(), shadow_frame->GetDexPC(), called_method); - jit->AddSamples(self, shadow_frame->GetMethod(), 1, /*with_backedges*/false); } return !self->IsExceptionPending(); } diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc index 2c7282115f..ce06a036ec 100644 --- a/runtime/interpreter/unstarted_runtime.cc +++ b/runtime/interpreter/unstarted_runtime.cc @@ -1438,7 +1438,11 @@ void UnstartedRuntime::UnstartedUnsafeCompareAndSwapObject( mirror::HeapReference<mirror::Object>* field_addr = reinterpret_cast<mirror::HeapReference<mirror::Object>*>( reinterpret_cast<uint8_t*>(obj) + static_cast<size_t>(offset)); - ReadBarrier::Barrier<mirror::Object, kWithReadBarrier, /* kAlwaysUpdateField */ true>( + ReadBarrier::Barrier< + mirror::Object, + /* kIsVolatile */ false, + kWithReadBarrier, + /* kAlwaysUpdateField */ true>( obj, MemberOffset(offset), field_addr); diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index 59373eb34e..47ace7fa71 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -47,13 +47,9 @@ namespace jit { static constexpr int kProtAll = PROT_READ | PROT_WRITE | PROT_EXEC; static constexpr int kProtData = PROT_READ | PROT_WRITE; static constexpr int kProtCode = PROT_READ | PROT_EXEC; -static constexpr int kProtReadOnly = PROT_READ; -static constexpr int kProtNone = PROT_NONE; static constexpr size_t kCodeSizeLogThreshold = 50 * KB; static constexpr size_t kStackMapSizeLogThreshold = 50 * KB; -static constexpr size_t kMinMapSpacingPages = 1; -static constexpr size_t kMaxMapSpacingPages = 128; #define CHECKED_MPROTECT(memory, size, prot) \ do { \ @@ -64,39 +60,12 @@ static constexpr size_t kMaxMapSpacingPages = 128; } \ } while (false) \ -static MemMap* SplitMemMap(MemMap* existing_map, - const char* name, - size_t split_offset, - int split_prot, - std::string* error_msg, - bool use_ashmem, - unique_fd* shmem_fd = nullptr) { - std::string error_str; - uint8_t* divider = existing_map->Begin() + split_offset; - MemMap* new_map = existing_map->RemapAtEnd(divider, - name, - split_prot, - MAP_SHARED, - &error_str, - use_ashmem, - shmem_fd); - if (new_map == nullptr) { - std::ostringstream oss; - oss << "Failed to create spacing for " << name << ": " - << error_str << " offset=" << split_offset; - *error_msg = oss.str(); - return nullptr; - } - return new_map; -} - JitCodeCache* JitCodeCache::Create(size_t initial_capacity, size_t max_capacity, bool generate_debug_info, std::string* error_msg) { ScopedTrace trace(__PRETTY_FUNCTION__); - CHECK_GT(max_capacity, initial_capacity); - CHECK_GE(max_capacity - kMaxMapSpacingPages * kPageSize, initial_capacity); + CHECK_GE(max_capacity, initial_capacity); // Generating debug information is for using the Linux perf tool on // host which does not work with ashmem. @@ -106,10 +75,6 @@ JitCodeCache* JitCodeCache::Create(size_t initial_capacity, // With 'perf', we want a 1-1 mapping between an address and a method. bool garbage_collect_code = !generate_debug_info; - // We only use two mappings (separating rw from rx) if we are able to use ashmem. - // See the above comment for debug information and not using ashmem. - bool use_two_mappings = use_ashmem; - // We need to have 32 bit offsets from method headers in code cache which point to things // in the data cache. If the maps are more than 4G apart, having multiple maps wouldn't work. // Ensure we're below 1 GB to be safe. @@ -121,10 +86,6 @@ JitCodeCache* JitCodeCache::Create(size_t initial_capacity, return nullptr; } - // Align both capacities to page size, as that's the unit mspaces use. - initial_capacity = RoundDown(initial_capacity, 2 * kPageSize); - max_capacity = RoundDown(max_capacity, 2 * kPageSize); - std::string error_str; // Map name specific for android_os_Debug.cpp accounting. // Map in low 4gb to simplify accessing root tables for x86_64. @@ -146,138 +107,35 @@ JitCodeCache* JitCodeCache::Create(size_t initial_capacity, return nullptr; } - // Create a region for JIT data and executable code. This will be - // laid out as: - // - // +----------------+ -------------------- - // | code_sync_map_ | ^ code_sync_size ^ - // | | v | - // +----------------+ -- | - // : : ^ | - // : post_code_map : | post_code_size | - // : [padding] : v | - // +----------------+ - | - // | | ^ | - // | code_map | | code_size | total_mapping_size - // | [JIT Code] | v | - // +----------------+ - | - // : : ^ | - // : pre_code_map : | pre_code_size | - // : [padding] : v | - // +----------------+ - | - // | | ^ | - // | data_map | | data_size | - // | [Jit Data] | v v - // +----------------+ -------------------- - // - // The code_sync_map_ contains a page that we use flush CPU instruction - // pipelines (see FlushInstructionPipelines()). - // - // The padding regions - pre_code_map and post_code_map - exist to - // put some random distance between the writable JIT code mapping - // and the executable mapping. The padding is discarded at the end - // of this function. - // - size_t data_size = (max_capacity - kMaxMapSpacingPages * kPageSize) / 2; - size_t pre_code_size = - GetRandomNumber(kMinMapSpacingPages, kMaxMapSpacingPages - 1) * kPageSize; - size_t code_size = max_capacity - data_size - kMaxMapSpacingPages * kPageSize; - size_t code_sync_size = kPageSize; - size_t post_code_size = kMaxMapSpacingPages * kPageSize - pre_code_size - code_sync_size; - DCHECK_EQ(data_size, code_size); - DCHECK_EQ(pre_code_size + post_code_size + code_sync_size, kMaxMapSpacingPages * kPageSize); - DCHECK_EQ(data_size + pre_code_size + code_size + post_code_size + code_sync_size, max_capacity); - - // Create pre-code padding region after data region, discarded after - // code and data regions are set-up. - std::unique_ptr<MemMap> pre_code_map(SplitMemMap(data_map.get(), - "jit-code-cache-padding", - data_size, - kProtNone, - error_msg, - use_ashmem)); - if (pre_code_map == nullptr) { - return nullptr; - } - DCHECK_EQ(data_map->Size(), data_size); - DCHECK_EQ(pre_code_map->Size(), pre_code_size + code_size + post_code_size + code_sync_size); - - // Create code region. - unique_fd writable_code_fd; - std::unique_ptr<MemMap> code_map(SplitMemMap(pre_code_map.get(), - "jit-code-cache", - pre_code_size, - use_two_mappings ? kProtCode : kProtAll, - error_msg, - use_ashmem, - &writable_code_fd)); - if (code_map == nullptr) { - return nullptr; - } - DCHECK_EQ(pre_code_map->Size(), pre_code_size); - DCHECK_EQ(code_map->Size(), code_size + post_code_size + code_sync_size); - - // Padding after code region, discarded after code and data regions - // are set-up. - std::unique_ptr<MemMap> post_code_map(SplitMemMap(code_map.get(), - "jit-code-cache-padding", - code_size, - kProtNone, - error_msg, - use_ashmem)); - if (post_code_map == nullptr) { - return nullptr; - } - DCHECK_EQ(code_map->Size(), code_size); - DCHECK_EQ(post_code_map->Size(), post_code_size + code_sync_size); + // Align both capacities to page size, as that's the unit mspaces use. + initial_capacity = RoundDown(initial_capacity, 2 * kPageSize); + max_capacity = RoundDown(max_capacity, 2 * kPageSize); + + // Data cache is 1 / 2 of the map. + // TODO: Make this variable? + size_t data_size = max_capacity / 2; + size_t code_size = max_capacity - data_size; + DCHECK_EQ(code_size + data_size, max_capacity); + uint8_t* divider = data_map->Begin() + data_size; - std::unique_ptr<MemMap> code_sync_map(SplitMemMap(post_code_map.get(), - "jit-code-sync", - post_code_size, - kProtCode, - error_msg, - use_ashmem)); - if (code_sync_map == nullptr) { + MemMap* code_map = + data_map->RemapAtEnd(divider, "jit-code-cache", kProtAll, &error_str, use_ashmem); + if (code_map == nullptr) { + std::ostringstream oss; + oss << "Failed to create read write execute cache: " << error_str << " size=" << max_capacity; + *error_msg = oss.str(); return nullptr; } - DCHECK_EQ(post_code_map->Size(), post_code_size); - DCHECK_EQ(code_sync_map->Size(), code_sync_size); - - std::unique_ptr<MemMap> writable_code_map; - if (use_two_mappings) { - // Allocate the R/W view. - writable_code_map.reset(MemMap::MapFile(code_size, - kProtData, - MAP_SHARED, - writable_code_fd.get(), - /* start */ 0, - /* low_4gb */ true, - "jit-writable-code", - &error_str)); - if (writable_code_map == nullptr) { - std::ostringstream oss; - oss << "Failed to create writable code cache: " << error_str << " size=" << code_size; - *error_msg = oss.str(); - return nullptr; - } - } + DCHECK_EQ(code_map->Begin(), divider); data_size = initial_capacity / 2; code_size = initial_capacity - data_size; DCHECK_EQ(code_size + data_size, initial_capacity); - return new JitCodeCache(writable_code_map.release(), - code_map.release(), - data_map.release(), - code_sync_map.release(), - code_size, - data_size, - max_capacity, - garbage_collect_code); + return new JitCodeCache( + code_map, data_map.release(), code_size, data_size, max_capacity, garbage_collect_code); } -JitCodeCache::JitCodeCache(MemMap* writable_code_map, - MemMap* executable_code_map, +JitCodeCache::JitCodeCache(MemMap* code_map, MemMap* data_map, - MemMap* code_sync_map, size_t initial_code_capacity, size_t initial_data_capacity, size_t max_capacity, @@ -285,10 +143,8 @@ JitCodeCache::JitCodeCache(MemMap* writable_code_map, : lock_("Jit code cache", kJitCodeCacheLock), lock_cond_("Jit code cache condition variable", lock_), collection_in_progress_(false), + code_map_(code_map), data_map_(data_map), - executable_code_map_(executable_code_map), - writable_code_map_(writable_code_map), - code_sync_map_(code_sync_map), max_capacity_(max_capacity), current_capacity_(initial_code_capacity + initial_data_capacity), code_end_(initial_code_capacity), @@ -308,8 +164,7 @@ JitCodeCache::JitCodeCache(MemMap* writable_code_map, inline_cache_cond_("Jit inline cache condition variable", lock_) { DCHECK_GE(max_capacity, initial_code_capacity + initial_data_capacity); - MemMap* writable_map = GetWritableMemMap(); - code_mspace_ = create_mspace_with_base(writable_map->Begin(), code_end_, false /*locked*/); + code_mspace_ = create_mspace_with_base(code_map_->Begin(), code_end_, false /*locked*/); data_mspace_ = create_mspace_with_base(data_map_->Begin(), data_end_, false /*locked*/); if (code_mspace_ == nullptr || data_mspace_ == nullptr) { @@ -318,10 +173,7 @@ JitCodeCache::JitCodeCache(MemMap* writable_code_map, SetFootprintLimit(current_capacity_); - if (writable_code_map_ != nullptr) { - CHECKED_MPROTECT(writable_code_map_->Begin(), writable_code_map_->Size(), kProtReadOnly); - } - CHECKED_MPROTECT(executable_code_map_->Begin(), executable_code_map_->Size(), kProtCode); + CHECKED_MPROTECT(code_map_->Begin(), code_map_->Size(), kProtCode); CHECKED_MPROTECT(data_map_->Begin(), data_map_->Size(), kProtData); VLOG(jit) << "Created jit code cache: initial data size=" @@ -331,7 +183,7 @@ JitCodeCache::JitCodeCache(MemMap* writable_code_map, } bool JitCodeCache::ContainsPc(const void* ptr) const { - return executable_code_map_->Begin() <= ptr && ptr < executable_code_map_->End(); + return code_map_->Begin() <= ptr && ptr < code_map_->End(); } bool JitCodeCache::ContainsMethod(ArtMethod* method) { @@ -344,96 +196,27 @@ bool JitCodeCache::ContainsMethod(ArtMethod* method) { return false; } -/* This method is only for CHECK/DCHECK that pointers are within to a region. */ -static bool IsAddressInMap(const void* addr, - const MemMap* mem_map, - const char* check_name) { - if (addr == nullptr || mem_map->HasAddress(addr)) { - return true; - } - LOG(ERROR) << "Is" << check_name << "Address " << addr - << " not in [" << reinterpret_cast<void*>(mem_map->Begin()) - << ", " << reinterpret_cast<void*>(mem_map->Begin() + mem_map->Size()) << ")"; - return false; -} - -bool JitCodeCache::IsDataAddress(const void* raw_addr) const { - return IsAddressInMap(raw_addr, data_map_.get(), "Data"); -} - -bool JitCodeCache::IsExecutableAddress(const void* raw_addr) const { - return IsAddressInMap(raw_addr, executable_code_map_.get(), "Executable"); -} - -bool JitCodeCache::IsWritableAddress(const void* raw_addr) const { - return IsAddressInMap(raw_addr, GetWritableMemMap(), "Writable"); -} - -// Convert one address within the source map to the same offset within the destination map. -static void* ConvertAddress(const void* source_address, - const MemMap* source_map, - const MemMap* destination_map) { - DCHECK(source_map->HasAddress(source_address)) << source_address; - ptrdiff_t offset = reinterpret_cast<const uint8_t*>(source_address) - source_map->Begin(); - uintptr_t address = reinterpret_cast<uintptr_t>(destination_map->Begin()) + offset; - return reinterpret_cast<void*>(address); -} - -template <typename T> -T* JitCodeCache::ToExecutableAddress(T* writable_address) const { - CHECK(IsWritableAddress(writable_address)); - if (writable_address == nullptr) { - return nullptr; - } - void* executable_address = ConvertAddress(writable_address, - GetWritableMemMap(), - executable_code_map_.get()); - CHECK(IsExecutableAddress(executable_address)); - return reinterpret_cast<T*>(executable_address); -} - -void* JitCodeCache::ToWritableAddress(const void* executable_address) const { - CHECK(IsExecutableAddress(executable_address)); - if (executable_address == nullptr) { - return nullptr; - } - void* writable_address = ConvertAddress(executable_address, - executable_code_map_.get(), - GetWritableMemMap()); - CHECK(IsWritableAddress(writable_address)); - return writable_address; -} - class ScopedCodeCacheWrite : ScopedTrace { public: - explicit ScopedCodeCacheWrite(JitCodeCache* code_cache) - : ScopedTrace("ScopedCodeCacheWrite") { + explicit ScopedCodeCacheWrite(MemMap* code_map, bool only_for_tlb_shootdown = false) + : ScopedTrace("ScopedCodeCacheWrite"), + code_map_(code_map), + only_for_tlb_shootdown_(only_for_tlb_shootdown) { ScopedTrace trace("mprotect all"); - int prot_to_start_writing = kProtAll; - if (code_cache->writable_code_map_ == nullptr) { - // If there is only one mapping, use the executable mapping and toggle between rwx and rx. - prot_to_start_writing = kProtAll; - prot_to_stop_writing_ = kProtCode; - } else { - // If there are two mappings, use the writable mapping and toggle between rw and r. - prot_to_start_writing = kProtData; - prot_to_stop_writing_ = kProtReadOnly; - } - writable_map_ = code_cache->GetWritableMemMap(); - // If we're using ScopedCacheWrite only for TLB shootdown, we limit the scope of mprotect to - // one page. - size_ = writable_map_->Size(); - CHECKED_MPROTECT(writable_map_->Begin(), size_, prot_to_start_writing); + CHECKED_MPROTECT( + code_map_->Begin(), only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), kProtAll); } ~ScopedCodeCacheWrite() { ScopedTrace trace("mprotect code"); - CHECKED_MPROTECT(writable_map_->Begin(), size_, prot_to_stop_writing_); + CHECKED_MPROTECT( + code_map_->Begin(), only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), kProtCode); } - private: - int prot_to_stop_writing_; - MemMap* writable_map_; - size_t size_; + MemMap* const code_map_; + + // If we're using ScopedCacheWrite only for TLB shootdown, we limit the scope of mprotect to + // one page. + const bool only_for_tlb_shootdown_; DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite); }; @@ -448,6 +231,7 @@ uint8_t* JitCodeCache::CommitCode(Thread* self, size_t fp_spill_mask, const uint8_t* code, size_t code_size, + size_t data_size, bool osr, Handle<mirror::ObjectArray<mirror::Object>> roots, bool has_should_deoptimize_flag, @@ -462,6 +246,7 @@ uint8_t* JitCodeCache::CommitCode(Thread* self, fp_spill_mask, code, code_size, + data_size, osr, roots, has_should_deoptimize_flag, @@ -479,6 +264,7 @@ uint8_t* JitCodeCache::CommitCode(Thread* self, fp_spill_mask, code, code_size, + data_size, osr, roots, has_should_deoptimize_flag, @@ -540,10 +326,8 @@ static void FillRootTable(uint8_t* roots_data, Handle<mirror::ObjectArray<mirror } } -uint8_t* JitCodeCache::GetRootTable(const void* code_ptr, uint32_t* number_of_roots) { - CHECK(IsExecutableAddress(code_ptr)); +static uint8_t* GetRootTable(const void* code_ptr, uint32_t* number_of_roots = nullptr) { OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); - // GetOptimizedCodeInfoPtr uses offsets relative to the EXECUTABLE address. uint8_t* data = method_header->GetOptimizedCodeInfoPtr(); uint32_t roots = GetNumberOfRoots(data); if (number_of_roots != nullptr) { @@ -588,8 +372,6 @@ static inline void ProcessWeakClass(GcRoot<mirror::Class>* root_ptr, void JitCodeCache::SweepRootTables(IsMarkedVisitor* visitor) { MutexLock mu(Thread::Current(), lock_); for (const auto& entry : method_code_map_) { - // GetRootTable takes an EXECUTABLE address. - CHECK(IsExecutableAddress(entry.first)); uint32_t number_of_roots = 0; uint8_t* roots_data = GetRootTable(entry.first, &number_of_roots); GcRoot<mirror::Object>* roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data); @@ -627,19 +409,17 @@ void JitCodeCache::SweepRootTables(IsMarkedVisitor* visitor) { } } -void JitCodeCache::FreeCodeAndData(const void* code_ptr) { - CHECK(IsExecutableAddress(code_ptr)); +void JitCodeCache::FreeCode(const void* code_ptr) { + uintptr_t allocation = FromCodeToAllocation(code_ptr); // Notify native debugger that we are about to remove the code. // It does nothing if we are not using native debugger. DeleteJITCodeEntryForAddress(reinterpret_cast<uintptr_t>(code_ptr)); - // GetRootTable takes an EXECUTABLE address. FreeData(GetRootTable(code_ptr)); - FreeRawCode(reinterpret_cast<uint8_t*>(FromCodeToAllocation(code_ptr))); + FreeCode(reinterpret_cast<uint8_t*>(allocation)); } void JitCodeCache::FreeAllMethodHeaders( const std::unordered_set<OatQuickMethodHeader*>& method_headers) { - // method_headers are expected to be in the executable region. { MutexLock mu(Thread::Current(), *Locks::cha_lock_); Runtime::Current()->GetClassLinker()->GetClassHierarchyAnalysis() @@ -651,9 +431,9 @@ void JitCodeCache::FreeAllMethodHeaders( // so it's possible for the same method_header to start representing // different compile code. MutexLock mu(Thread::Current(), lock_); - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(code_map_.get()); for (const OatQuickMethodHeader* method_header : method_headers) { - FreeCodeAndData(method_header->GetCode()); + FreeCode(method_header->GetCode()); } } @@ -670,10 +450,9 @@ void JitCodeCache::RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) { // with the classlinker_classes_lock_ held, and suspending ourselves could // lead to a deadlock. { - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(code_map_.get()); for (auto it = method_code_map_.begin(); it != method_code_map_.end();) { if (alloc.ContainsUnsafe(it->second)) { - CHECK(IsExecutableAddress(OatQuickMethodHeader::FromCodePointer(it->first))); method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->first)); it = method_code_map_.erase(it); } else { @@ -765,129 +544,6 @@ static void ClearMethodCounter(ArtMethod* method, bool was_warm) { method->SetCounter(std::min(jit_warmup_threshold - 1, 1)); } -static void FlushInstructionPiplines(uint8_t* sync_page) { - // After updating the JIT code cache we need to force all CPUs to - // flush their instruction pipelines. In the absence of system call - // to do this explicitly, we can achieve this indirectly by toggling - // permissions on an executable page. This should send an IPI to - // each core to update the TLB entry with the interrupt raised on - // each core causing the instruction pipeline to be flushed. - CHECKED_MPROTECT(sync_page, kPageSize, kProtAll); - // Ensure the sync_page is present otherwise a TLB update may not be - // necessary. - sync_page[0] = 0; - CHECKED_MPROTECT(sync_page, kPageSize, kProtCode); -} - -#ifdef __aarch64__ - -static void FlushJitCodeCacheRange(uint8_t* code_ptr, - uint8_t* writable_ptr, - size_t code_size) { - // Cache maintenance instructions can cause permission faults when a - // page is not present (e.g. swapped out or not backed). These - // faults should be handled by the kernel, but a bug in some Linux - // kernels may surface these permission faults to user-land which - // does not currently deal with them (b/63885946). To work around - // this, we read a value from each page to fault it in before - // attempting to perform cache maintenance operations. - // - // For reference, this behavior is caused by this commit: - // https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c - - // The cache-line size could be probed for from the CPU, but - // assuming a safe lower bound is safe for CPUs that have different - // cache-line sizes for big and little cores. - static const uintptr_t kSafeCacheLineSize = 32; - - // Ensure stores are present in L1 data cache. - __asm __volatile("dsb ish" ::: "memory"); - - volatile uint8_t mutant; - - // Push dirty cache-lines out to the point of unification (PoU). The - // point of unification is the first point in the cache/memory - // hierarchy where the instruction cache and data cache have the - // same view of memory. The PoU is where an instruction fetch will - // fetch the new code generated by the JIT. - // - // See: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch11s04.html - uintptr_t writable_addr = RoundDown(reinterpret_cast<uintptr_t>(writable_ptr), - kSafeCacheLineSize); - uintptr_t writable_end = RoundUp(reinterpret_cast<uintptr_t>(writable_ptr) + code_size, - kSafeCacheLineSize); - while (writable_addr < writable_end) { - // Read from the cache-line to minimize the chance that a cache - // maintenance instruction causes a fault (see kernel bug comment - // above). - mutant = *reinterpret_cast<const uint8_t*>(writable_addr); - - // Flush cache-line - __asm volatile("dc cvau, %0" :: "r"(writable_addr) : "memory"); - writable_addr += kSafeCacheLineSize; - } - - __asm __volatile("dsb ish" ::: "memory"); - - uintptr_t code_addr = RoundDown(reinterpret_cast<uintptr_t>(code_ptr), kSafeCacheLineSize); - const uintptr_t code_end = RoundUp(reinterpret_cast<uintptr_t>(code_ptr) + code_size, - kSafeCacheLineSize); - while (code_addr < code_end) { - // Read from the cache-line to minimize the chance that a cache - // maintenance instruction causes a fault (see kernel bug comment - // above). - mutant = *reinterpret_cast<const uint8_t*>(code_addr); - - // Invalidating the data cache line is only strictly necessary - // when the JIT code cache has two mappings (the default). We know - // this cache line is clean so this is just invalidating it (using - // "dc ivac" would be preferable, but counts as a write and this - // memory may not be mapped write permission). - __asm volatile("dc cvau, %0" :: "r"(code_addr) : "memory"); - - // Invalidate the instruction cache line to force instructions in - // range to be re-fetched following update. - __asm volatile("ic ivau, %0" :: "r"(code_addr) : "memory"); - - code_addr += kSafeCacheLineSize; - } - - // Wait for code cache invalidations to complete. - __asm __volatile("dsb ish" ::: "memory"); - - // Reset fetched instruction stream. - __asm __volatile("isb"); -} - -#else // __aarch64 - -static void FlushJitCodeCacheRange(uint8_t* code_ptr, - uint8_t* writable_ptr, - size_t code_size) { - if (writable_ptr != code_ptr) { - // When there are two mappings of the JIT code cache, RX and - // RW, flush the RW version first as we've just dirtied the - // cache lines with new code. Flushing the RX version first - // can cause a permission fault as the those addresses are not - // writable, but can appear dirty in the cache. There is a lot - // of potential subtlety here depending on how the cache is - // indexed and tagged. - // - // Flushing the RX version after the RW version is just - // invalidating cachelines in the instruction cache. This is - // necessary as the instruction cache will often have a - // different set of cache lines present and because the JIT - // code cache can start a new function at any boundary within - // a cache-line. - FlushDataCache(reinterpret_cast<char*>(writable_ptr), - reinterpret_cast<char*>(writable_ptr + code_size)); - } - FlushInstructionCache(reinterpret_cast<char*>(code_ptr), - reinterpret_cast<char*>(code_ptr + code_size)); -} - -#endif // __aarch64 - uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, ArtMethod* method, uint8_t* stack_map, @@ -898,6 +554,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, size_t fp_spill_mask, const uint8_t* code, size_t code_size, + size_t data_size, bool osr, Handle<mirror::ObjectArray<mirror::Object>> roots, bool has_should_deoptimize_flag, @@ -917,37 +574,35 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, MutexLock mu(self, lock_); WaitForPotentialCollectionToComplete(self); { - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(code_map_.get()); memory = AllocateCode(total_size); if (memory == nullptr) { return nullptr; } - uint8_t* writable_ptr = memory + header_size; - code_ptr = ToExecutableAddress(writable_ptr); - - std::copy(code, code + code_size, writable_ptr); - OatQuickMethodHeader* writable_method_header = - OatQuickMethodHeader::FromCodePointer(writable_ptr); - // We need to be able to write the OatQuickMethodHeader, so we use writable_method_header. - // Otherwise, the offsets encoded in OatQuickMethodHeader are used relative to an executable - // address, so we use code_ptr. - new (writable_method_header) OatQuickMethodHeader( + code_ptr = memory + header_size; + + std::copy(code, code + code_size, code_ptr); + method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); + new (method_header) OatQuickMethodHeader( code_ptr - stack_map, code_ptr - method_info, frame_size_in_bytes, core_spill_mask, fp_spill_mask, code_size); - - FlushJitCodeCacheRange(code_ptr, writable_ptr, code_size); - FlushInstructionPiplines(code_sync_map_->Begin()); - + // Flush caches before we remove write permission because some ARMv8 Qualcomm kernels may + // trigger a segfault if a page fault occurs when requesting a cache maintenance operation. + // This is a kernel bug that we need to work around until affected devices (e.g. Nexus 5X and + // 6P) stop being supported or their kernels are fixed. + // + // For reference, this behavior is caused by this commit: + // https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c + FlushInstructionCache(reinterpret_cast<char*>(code_ptr), + reinterpret_cast<char*>(code_ptr + code_size)); DCHECK(!Runtime::Current()->IsAotCompiler()); if (has_should_deoptimize_flag) { - writable_method_header->SetHasShouldDeoptimizeFlag(); + method_header->SetHasShouldDeoptimizeFlag(); } - // All the pointers exported from the cache are executable addresses. - method_header = ToExecutableAddress(writable_method_header); } number_of_compilations_++; @@ -986,14 +641,16 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, // but below we still make the compiled code valid for the method. MutexLock mu(self, lock_); // Fill the root table before updating the entry point. - CHECK(IsDataAddress(roots_data)); DCHECK_EQ(FromStackMapToRoots(stack_map), roots_data); DCHECK_LE(roots_data, stack_map); FillRootTable(roots_data, roots); - - // Ensure the updates to the root table are visible with a store fence. - QuasiAtomic::ThreadFenceSequentiallyConsistent(); - + { + // Flush data cache, as compiled code references literals in it. + // We also need a TLB shootdown to act as memory barrier across cores. + ScopedCodeCacheWrite ccw(code_map_.get(), /* only_for_tlb_shootdown */ true); + FlushDataCache(reinterpret_cast<char*>(roots_data), + reinterpret_cast<char*>(roots_data + data_size)); + } method_code_map_.Put(code_ptr, method); if (osr) { number_of_osr_compilations_++; @@ -1041,11 +698,11 @@ bool JitCodeCache::RemoveMethod(ArtMethod* method, bool release_memory) { bool in_cache = false; { - ScopedCodeCacheWrite ccw(this); + ScopedCodeCacheWrite ccw(code_map_.get()); for (auto code_iter = method_code_map_.begin(); code_iter != method_code_map_.end();) { if (code_iter->second == method) { if (release_memory) { - FreeCodeAndData(code_iter->first); + FreeCode(code_iter->first); } code_iter = method_code_map_.erase(code_iter); in_cache = true; @@ -1099,10 +756,10 @@ void JitCodeCache::NotifyMethodRedefined(ArtMethod* method) { profiling_infos_.erase(profile); } method->SetProfilingInfo(nullptr); - ScopedCodeCacheWrite ccw(this); + ScopedCodeCacheWrite ccw(code_map_.get()); for (auto code_iter = method_code_map_.begin(); code_iter != method_code_map_.end();) { if (code_iter->second == method) { - FreeCodeAndData(code_iter->first); + FreeCode(code_iter->first); code_iter = method_code_map_.erase(code_iter); continue; } @@ -1168,7 +825,6 @@ void JitCodeCache::ClearData(Thread* self, uint8_t* stack_map_data, uint8_t* roots_data) { DCHECK_EQ(FromStackMapToRoots(stack_map_data), roots_data); - CHECK(IsDataAddress(roots_data)); MutexLock mu(self, lock_); FreeData(reinterpret_cast<uint8_t*>(roots_data)); } @@ -1290,11 +946,11 @@ void JitCodeCache::NotifyCollectionDone(Thread* self) { void JitCodeCache::SetFootprintLimit(size_t new_footprint) { size_t per_space_footprint = new_footprint / 2; - CHECK(IsAlignedParam(per_space_footprint, kPageSize)); + DCHECK(IsAlignedParam(per_space_footprint, kPageSize)); DCHECK_EQ(per_space_footprint * 2, new_footprint); mspace_set_footprint_limit(data_mspace_, per_space_footprint); { - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(code_map_.get()); mspace_set_footprint_limit(code_mspace_, per_space_footprint); } } @@ -1315,9 +971,7 @@ bool JitCodeCache::IncreaseCodeCacheCapacity() { current_capacity_ = max_capacity_; } - if (!kIsDebugBuild || VLOG_IS_ON(jit)) { - LOG(INFO) << "Increasing code cache capacity to " << PrettySize(current_capacity_); - } + VLOG(jit) << "Increasing code cache capacity to " << PrettySize(current_capacity_); SetFootprintLimit(current_capacity_); @@ -1372,8 +1026,8 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { number_of_collections_++; live_bitmap_.reset(CodeCacheBitmap::Create( "code-cache-bitmap", - reinterpret_cast<uintptr_t>(executable_code_map_->Begin()), - reinterpret_cast<uintptr_t>(executable_code_map_->Begin() + current_capacity_ / 2))); + reinterpret_cast<uintptr_t>(code_map_->Begin()), + reinterpret_cast<uintptr_t>(code_map_->Begin() + current_capacity_ / 2))); collection_in_progress_ = true; } } @@ -1388,21 +1042,17 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { do_full_collection = ShouldDoFullCollection(); } - if (!kIsDebugBuild || VLOG_IS_ON(jit)) { - LOG(INFO) << "Do " - << (do_full_collection ? "full" : "partial") - << " code cache collection, code=" - << PrettySize(CodeCacheSize()) - << ", data=" << PrettySize(DataCacheSize()); - } + VLOG(jit) << "Do " + << (do_full_collection ? "full" : "partial") + << " code cache collection, code=" + << PrettySize(CodeCacheSize()) + << ", data=" << PrettySize(DataCacheSize()); DoCollection(self, /* collect_profiling_info */ do_full_collection); - if (!kIsDebugBuild || VLOG_IS_ON(jit)) { - LOG(INFO) << "After code cache collection, code=" - << PrettySize(CodeCacheSize()) - << ", data=" << PrettySize(DataCacheSize()); - } + VLOG(jit) << "After code cache collection, code=" + << PrettySize(CodeCacheSize()) + << ", data=" << PrettySize(DataCacheSize()); { MutexLock mu(self, lock_); @@ -1449,16 +1099,14 @@ void JitCodeCache::RemoveUnmarkedCode(Thread* self) { std::unordered_set<OatQuickMethodHeader*> method_headers; { MutexLock mu(self, lock_); - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(code_map_.get()); // Iterate over all compiled code and remove entries that are not marked. for (auto it = method_code_map_.begin(); it != method_code_map_.end();) { const void* code_ptr = it->first; - CHECK(IsExecutableAddress(code_ptr)); uintptr_t allocation = FromCodeToAllocation(code_ptr); if (GetLiveBitmap()->Test(allocation)) { ++it; } else { - CHECK(IsExecutableAddress(it->first)); method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->first)); it = method_code_map_.erase(it); } @@ -1501,7 +1149,6 @@ void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { for (const auto& it : method_code_map_) { ArtMethod* method = it.second; const void* code_ptr = it.first; - CHECK(IsExecutableAddress(code_ptr)); const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); if (method_header->GetEntryPoint() == method->GetEntryPointFromQuickCompiledCode()) { GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(code_ptr)); @@ -1527,7 +1174,6 @@ void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { // Free all profiling infos of methods not compiled nor being compiled. auto profiling_kept_end = std::remove_if(profiling_infos_.begin(), profiling_infos_.end(), [this] (ProfilingInfo* info) NO_THREAD_SAFETY_ANALYSIS { - CHECK(IsDataAddress(info)); const void* ptr = info->GetMethod()->GetEntryPointFromQuickCompiledCode(); // We have previously cleared the ProfilingInfo pointer in the ArtMethod in the hope // that the compiled code would not get revived. As mutator threads run concurrently, @@ -1588,7 +1234,6 @@ OatQuickMethodHeader* JitCodeCache::LookupMethodHeader(uintptr_t pc, ArtMethod* --it; const void* code_ptr = it->first; - CHECK(IsExecutableAddress(code_ptr)); OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); if (!method_header->Contains(pc)) { return nullptr; @@ -1671,7 +1316,6 @@ ProfilingInfo* JitCodeCache::AddProfilingInfoInternal(Thread* self ATTRIBUTE_UNU // store in the ArtMethod's ProfilingInfo pointer. QuasiAtomic::ThreadFenceRelease(); - CHECK(IsDataAddress(info)); method->SetProfilingInfo(info); profiling_infos_.push_back(info); histogram_profiling_info_memory_use_.AddValue(profile_info_size); @@ -1684,8 +1328,7 @@ void* JitCodeCache::MoreCore(const void* mspace, intptr_t increment) NO_THREAD_S if (code_mspace_ == mspace) { size_t result = code_end_; code_end_ += increment; - MemMap* writable_map = GetWritableMemMap(); - return reinterpret_cast<void*>(result + writable_map->Begin()); + return reinterpret_cast<void*>(result + code_map_->Begin()); } else { DCHECK_EQ(data_mspace_, mspace); size_t result = data_end_; @@ -1837,7 +1480,6 @@ void JitCodeCache::DoneCompiling(ArtMethod* method, Thread* self ATTRIBUTE_UNUSE size_t JitCodeCache::GetMemorySizeOfCodePointer(const void* ptr) { MutexLock mu(Thread::Current(), lock_); - CHECK(IsExecutableAddress(ptr)); return mspace_usable_size(reinterpret_cast<const void*>(FromCodeToAllocation(ptr))); } @@ -1873,27 +1515,22 @@ uint8_t* JitCodeCache::AllocateCode(size_t code_size) { size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); // Ensure the header ends up at expected instruction alignment. DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(result + header_size), alignment); - CHECK(IsWritableAddress(result)); used_memory_for_code_ += mspace_usable_size(result); return result; } -void JitCodeCache::FreeRawCode(void* code) { - CHECK(IsExecutableAddress(code)); - void* writable_code = ToWritableAddress(code); - used_memory_for_code_ -= mspace_usable_size(writable_code); - mspace_free(code_mspace_, writable_code); +void JitCodeCache::FreeCode(uint8_t* code) { + used_memory_for_code_ -= mspace_usable_size(code); + mspace_free(code_mspace_, code); } uint8_t* JitCodeCache::AllocateData(size_t data_size) { void* result = mspace_malloc(data_mspace_, data_size); - CHECK(IsDataAddress(reinterpret_cast<uint8_t*>(result))); used_memory_for_data_ += mspace_usable_size(result); return reinterpret_cast<uint8_t*>(result); } void JitCodeCache::FreeData(uint8_t* data) { - CHECK(IsDataAddress(data)); used_memory_for_data_ -= mspace_usable_size(data); mspace_free(data_mspace_, data); } diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h index 175501f915..daa1d616a6 100644 --- a/runtime/jit/jit_code_cache.h +++ b/runtime/jit/jit_code_cache.h @@ -113,6 +113,7 @@ class JitCodeCache { size_t fp_spill_mask, const uint8_t* code, size_t code_size, + size_t data_size, bool osr, Handle<mirror::ObjectArray<mirror::Object>> roots, bool has_should_deoptimize_flag, @@ -228,8 +229,6 @@ class JitCodeCache { REQUIRES(!lock_) REQUIRES_SHARED(Locks::mutator_lock_); - uint8_t* GetRootTable(const void* code_ptr, uint32_t* number_of_roots = nullptr); - // The GC needs to disallow the reading of inline caches when it processes them, // to avoid having a class being used while it is being deleted. void AllowInlineCacheAccess() REQUIRES(!lock_); @@ -248,13 +247,9 @@ class JitCodeCache { } private: - friend class ScopedCodeCacheWrite; - // Take ownership of maps. JitCodeCache(MemMap* code_map, MemMap* data_map, - MemMap* writable_code_map, - MemMap* code_sync_map, size_t initial_code_capacity, size_t initial_data_capacity, size_t max_capacity, @@ -272,6 +267,7 @@ class JitCodeCache { size_t fp_spill_mask, const uint8_t* code, size_t code_size, + size_t data_size, bool osr, Handle<mirror::ObjectArray<mirror::Object>> roots, bool has_should_deoptimize_flag, @@ -296,7 +292,7 @@ class JitCodeCache { REQUIRES(!Locks::cha_lock_); // Free in the mspace allocations for `code_ptr`. - void FreeCodeAndData(const void* code_ptr) REQUIRES(lock_); + void FreeCode(const void* code_ptr) REQUIRES(lock_); // Number of bytes allocated in the code cache. size_t CodeCacheSizeLocked() REQUIRES(lock_); @@ -329,7 +325,7 @@ class JitCodeCache { bool CheckLiveCompiledCodeHasProfilingInfo() REQUIRES(lock_); - void FreeRawCode(void* code) REQUIRES(lock_); + void FreeCode(uint8_t* code) REQUIRES(lock_); uint8_t* AllocateCode(size_t code_size) REQUIRES(lock_); void FreeData(uint8_t* data) REQUIRES(lock_); uint8_t* AllocateData(size_t data_size) REQUIRES(lock_); @@ -339,61 +335,25 @@ class JitCodeCache { REQUIRES(!lock_) REQUIRES_SHARED(Locks::mutator_lock_); - MemMap* GetWritableMemMap() const { - if (writable_code_map_ == nullptr) { - // The system required us to map the JIT Code Cache RWX (see - // JitCodeCache::Create()). - return executable_code_map_.get(); - } else { - // Executable code is mapped RX, and writable code is mapped RW - // to the underlying same memory, but at a different address. - return writable_code_map_.get(); - } - } - - bool IsDataAddress(const void* raw_addr) const; - - bool IsExecutableAddress(const void* raw_addr) const; - - bool IsWritableAddress(const void* raw_addr) const; - - template <typename T> - T* ToExecutableAddress(T* writable_address) const; - - void* ToWritableAddress(const void* executable_address) const; - // Lock for guarding allocations, collections, and the method_code_map_. Mutex lock_; // Condition to wait on during collection. ConditionVariable lock_cond_ GUARDED_BY(lock_); // Whether there is a code cache collection in progress. bool collection_in_progress_ GUARDED_BY(lock_); - // JITting methods obviously requires both write and execute permissions on a region of memory. - // In tye typical (non-debugging) case, we separate the memory mapped view that can write the code - // from a view that the runtime uses to execute the code. Having these two views eliminates any - // single address region having rwx permissions. An attacker could still write the writable - // address and then execute the executable address. We allocate the mappings with a random - // address relationship to each other which makes the attacker need two addresses rather than - // just one. In the debugging case there is no file descriptor to back the - // shared memory, and hence we have to use a single mapping. + // Mem map which holds code. + std::unique_ptr<MemMap> code_map_; // Mem map which holds data (stack maps and profiling info). std::unique_ptr<MemMap> data_map_; - // Mem map which holds a non-writable view of code for JIT. - std::unique_ptr<MemMap> executable_code_map_; - // Mem map which holds a non-executable view of code for JIT. - std::unique_ptr<MemMap> writable_code_map_; - // Mem map which holds one executable page that we use for flushing instruction - // fetch buffers. The code on this page is never executed. - std::unique_ptr<MemMap> code_sync_map_; // The opaque mspace for allocating code. void* code_mspace_ GUARDED_BY(lock_); // The opaque mspace for allocating data. void* data_mspace_ GUARDED_BY(lock_); // Bitmap for collecting code and data. std::unique_ptr<CodeCacheBitmap> live_bitmap_; - // Holds non-writable compiled code associated to the ArtMethod. + // Holds compiled code associated to the ArtMethod. SafeMap<const void*, ArtMethod*> method_code_map_ GUARDED_BY(lock_); - // Holds non-writable osr compiled code associated to the ArtMethod. + // Holds osr compiled code associated to the ArtMethod. SafeMap<ArtMethod*, const void*> osr_code_map_ GUARDED_BY(lock_); // ProfilingInfo objects we have allocated. std::vector<ProfilingInfo*> profiling_infos_ GUARDED_BY(lock_); diff --git a/runtime/mem_map.cc b/runtime/mem_map.cc index 4e82480aca..743604cc47 100644 --- a/runtime/mem_map.cc +++ b/runtime/mem_map.cc @@ -496,7 +496,7 @@ MemMap::~MemMap() { MEMORY_TOOL_MAKE_UNDEFINED(base_begin_, base_size_); int result = munmap(base_begin_, base_size_); if (result == -1) { - PLOG(FATAL) << "munmap failed: " << BaseBegin() << "..." << BaseEnd(); + PLOG(FATAL) << "munmap failed"; } } @@ -535,13 +535,8 @@ MemMap::MemMap(const std::string& name, uint8_t* begin, size_t size, void* base_ } } -MemMap* MemMap::RemapAtEnd(uint8_t* new_end, - const char* tail_name, - int tail_prot, - int sharing_flags, - std::string* error_msg, - bool use_ashmem, - unique_fd* shmem_fd) { +MemMap* MemMap::RemapAtEnd(uint8_t* new_end, const char* tail_name, int tail_prot, + std::string* error_msg, bool use_ashmem) { use_ashmem = use_ashmem && !kIsTargetLinux; DCHECK_GE(new_end, Begin()); DCHECK_LE(new_end, End()); @@ -560,12 +555,6 @@ MemMap* MemMap::RemapAtEnd(uint8_t* new_end, size_ = new_end - reinterpret_cast<uint8_t*>(begin_); base_size_ = new_base_end - reinterpret_cast<uint8_t*>(base_begin_); DCHECK_LE(begin_ + size_, reinterpret_cast<uint8_t*>(base_begin_) + base_size_); - if (base_size_ == 0u) { - // All pages in this MemMap have been handed out. Invalidate base - // pointer to prevent the destructor calling munmap() on - // zero-length region (which can't succeed). - base_begin_ = nullptr; - } size_t tail_size = old_end - new_end; uint8_t* tail_base_begin = new_base_end; size_t tail_base_size = old_base_end - new_base_end; @@ -573,14 +562,14 @@ MemMap* MemMap::RemapAtEnd(uint8_t* new_end, DCHECK_ALIGNED(tail_base_size, kPageSize); unique_fd fd; - int flags = MAP_ANONYMOUS | sharing_flags; + int flags = MAP_PRIVATE | MAP_ANONYMOUS; if (use_ashmem) { // android_os_Debug.cpp read_mapinfo assumes all ashmem regions associated with the VM are // prefixed "dalvik-". std::string debug_friendly_name("dalvik-"); debug_friendly_name += tail_name; fd.reset(ashmem_create_region(debug_friendly_name.c_str(), tail_base_size)); - flags = MAP_FIXED | sharing_flags; + flags = MAP_PRIVATE | MAP_FIXED; if (fd.get() == -1) { *error_msg = StringPrintf("ashmem_create_region failed for '%s': %s", tail_name, strerror(errno)); @@ -614,9 +603,6 @@ MemMap* MemMap::RemapAtEnd(uint8_t* new_end, fd.get()); return nullptr; } - if (shmem_fd != nullptr) { - shmem_fd->reset(fd.release()); - } return new MemMap(tail_name, actual, tail_size, actual, tail_base_size, tail_prot, false); } diff --git a/runtime/mem_map.h b/runtime/mem_map.h index d8908ad603..5603963eac 100644 --- a/runtime/mem_map.h +++ b/runtime/mem_map.h @@ -25,7 +25,6 @@ #include <string> #include "android-base/thread_annotations.h" -#include "android-base/unique_fd.h" namespace art { @@ -38,8 +37,6 @@ namespace art { #define USE_ART_LOW_4G_ALLOCATOR 0 #endif -using android::base::unique_fd; - #ifdef __linux__ static constexpr bool kMadviseZeroes = true; #else @@ -171,14 +168,11 @@ class MemMap { } // Unmap the pages at end and remap them to create another memory map. - // sharing_flags should be either MAP_PRIVATE or MAP_SHARED. MemMap* RemapAtEnd(uint8_t* new_end, const char* tail_name, int tail_prot, - int sharing_flags, std::string* error_msg, - bool use_ashmem = true, - unique_fd* shmem_fd = nullptr); + bool use_ashmem = true); static bool CheckNoGaps(MemMap* begin_map, MemMap* end_map) REQUIRES(!MemMap::mem_maps_lock_); diff --git a/runtime/mem_map_test.cc b/runtime/mem_map_test.cc index 99bf0045c4..a4ebb16d09 100644 --- a/runtime/mem_map_test.cc +++ b/runtime/mem_map_test.cc @@ -74,7 +74,6 @@ class MemMapTest : public CommonRuntimeTest { MemMap* m1 = m0->RemapAtEnd(base0 + page_size, "MemMapTest_RemapAtEndTest_map1", PROT_READ | PROT_WRITE, - MAP_PRIVATE, &error_msg); // Check the states of the two maps. EXPECT_EQ(m0->Begin(), base0) << error_msg; @@ -457,7 +456,6 @@ TEST_F(MemMapTest, AlignBy) { std::unique_ptr<MemMap> m1(m0->RemapAtEnd(base0 + 3 * page_size, "MemMapTest_AlignByTest_map1", PROT_READ | PROT_WRITE, - MAP_PRIVATE, &error_msg)); uint8_t* base1 = m1->Begin(); ASSERT_TRUE(base1 != nullptr) << error_msg; @@ -467,7 +465,6 @@ TEST_F(MemMapTest, AlignBy) { std::unique_ptr<MemMap> m2(m1->RemapAtEnd(base1 + 4 * page_size, "MemMapTest_AlignByTest_map2", PROT_READ | PROT_WRITE, - MAP_PRIVATE, &error_msg)); uint8_t* base2 = m2->Begin(); ASSERT_TRUE(base2 != nullptr) << error_msg; @@ -477,7 +474,6 @@ TEST_F(MemMapTest, AlignBy) { std::unique_ptr<MemMap> m3(m2->RemapAtEnd(base2 + 3 * page_size, "MemMapTest_AlignByTest_map1", PROT_READ | PROT_WRITE, - MAP_PRIVATE, &error_msg)); uint8_t* base3 = m3->Begin(); ASSERT_TRUE(base3 != nullptr) << error_msg; diff --git a/runtime/mirror/accessible_object.h b/runtime/mirror/accessible_object.h index a217193522..d489f14536 100644 --- a/runtime/mirror/accessible_object.h +++ b/runtime/mirror/accessible_object.h @@ -17,11 +17,8 @@ #ifndef ART_RUNTIME_MIRROR_ACCESSIBLE_OBJECT_H_ #define ART_RUNTIME_MIRROR_ACCESSIBLE_OBJECT_H_ -#include "class.h" -#include "gc_root.h" #include "object.h" #include "read_barrier_option.h" -#include "thread.h" namespace art { @@ -34,12 +31,6 @@ class MANAGED AccessibleObject : public Object { return OFFSET_OF_OBJECT_MEMBER(AccessibleObject, flag_); } - template<bool kTransactionActive> - void SetAccessible(bool value) REQUIRES_SHARED(Locks::mutator_lock_) { - UNUSED(padding_); - return SetFieldBoolean<kTransactionActive>(FlagOffset(), value ? 1u : 0u); - } - bool IsAccessible() REQUIRES_SHARED(Locks::mutator_lock_) { return GetFieldBoolean(FlagOffset()); } @@ -47,7 +38,7 @@ class MANAGED AccessibleObject : public Object { private: uint8_t flag_; // Padding required for correct alignment of subclasses like Executable, Field, etc. - uint8_t padding_[1]; + uint8_t padding_[1] ATTRIBUTE_UNUSED; DISALLOW_IMPLICIT_CONSTRUCTORS(AccessibleObject); }; diff --git a/runtime/mirror/array-inl.h b/runtime/mirror/array-inl.h index 63142d5c25..22812454d1 100644 --- a/runtime/mirror/array-inl.h +++ b/runtime/mirror/array-inl.h @@ -27,8 +27,7 @@ #include "class.h" #include "gc/heap-inl.h" #include "obj_ptr-inl.h" -#include "object-inl.h" -#include "thread.h" +#include "thread-current-inl.h" namespace art { namespace mirror { diff --git a/runtime/mirror/field-inl.h b/runtime/mirror/field-inl.h index d33df5c8e7..ad48202514 100644 --- a/runtime/mirror/field-inl.h +++ b/runtime/mirror/field-inl.h @@ -20,7 +20,8 @@ #include "field.h" #include "art_field-inl.h" -#include "mirror/dex_cache-inl.h" +#include "class-inl.h" +#include "dex_cache-inl.h" namespace art { @@ -87,6 +88,10 @@ inline void Field::SetType(ObjPtr<mirror::Class> type) { SetFieldObject<kTransactionActive>(OFFSET_OF_OBJECT_MEMBER(Field, type_), type); } +inline Primitive::Type Field::GetTypeAsPrimitiveType() { + return GetType()->GetPrimitiveType(); +} + } // namespace mirror } // namespace art diff --git a/runtime/mirror/field.h b/runtime/mirror/field.h index 40186a689b..6845575d18 100644 --- a/runtime/mirror/field.h +++ b/runtime/mirror/field.h @@ -20,8 +20,10 @@ #include "accessible_object.h" #include "base/enums.h" #include "gc_root.h" +#include "modifiers.h" #include "obj_ptr.h" #include "object.h" +#include "primitive.h" #include "read_barrier_option.h" namespace art { @@ -69,10 +71,7 @@ class MANAGED Field : public AccessibleObject { return (GetAccessFlags() & kAccVolatile) != 0; } - ALWAYS_INLINE Primitive::Type GetTypeAsPrimitiveType() - REQUIRES_SHARED(Locks::mutator_lock_) { - return GetType()->GetPrimitiveType(); - } + ALWAYS_INLINE Primitive::Type GetTypeAsPrimitiveType() REQUIRES_SHARED(Locks::mutator_lock_); mirror::Class* GetType() REQUIRES_SHARED(Locks::mutator_lock_) { return GetFieldObject<mirror::Class>(OFFSET_OF_OBJECT_MEMBER(Field, type_)); diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h index 086925b9c7..6eb200d0d3 100644 --- a/runtime/mirror/object-inl.h +++ b/runtime/mirror/object-inl.h @@ -393,14 +393,6 @@ inline size_t Object::SizeOf() { } template<VerifyObjectFlags kVerifyFlags, bool kIsVolatile> -inline uint8_t Object::GetFieldBoolean(MemberOffset field_offset) { - if (kVerifyFlags & kVerifyThis) { - VerifyObject(this); - } - return GetField<uint8_t, kIsVolatile>(field_offset); -} - -template<VerifyObjectFlags kVerifyFlags, bool kIsVolatile> inline int8_t Object::GetFieldByte(MemberOffset field_offset) { if (kVerifyFlags & kVerifyThis) { VerifyObject(this); @@ -724,11 +716,10 @@ inline T* Object::GetFieldObject(MemberOffset field_offset) { } uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); HeapReference<T>* objref_addr = reinterpret_cast<HeapReference<T>*>(raw_addr); - T* result = ReadBarrier::Barrier<T, kReadBarrierOption>(this, field_offset, objref_addr); - if (kIsVolatile) { - // TODO: Refactor to use a SequentiallyConsistent load instead. - QuasiAtomic::ThreadFenceAcquire(); // Ensure visibility of operations preceding store. - } + T* result = ReadBarrier::Barrier<T, kIsVolatile, kReadBarrierOption>( + this, + field_offset, + objref_addr); if (kVerifyFlags & kVerifyReads) { VerifyObject(result); } @@ -764,15 +755,7 @@ inline void Object::SetFieldObjectWithoutWriteBarrier(MemberOffset field_offset, } uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); HeapReference<Object>* objref_addr = reinterpret_cast<HeapReference<Object>*>(raw_addr); - if (kIsVolatile) { - // TODO: Refactor to use a SequentiallyConsistent store instead. - QuasiAtomic::ThreadFenceRelease(); // Ensure that prior accesses are visible before store. - objref_addr->Assign(new_value.Ptr()); - QuasiAtomic::ThreadFenceSequentiallyConsistent(); - // Ensure this store occurs before any volatile loads. - } else { - objref_addr->Assign(new_value.Ptr()); - } + objref_addr->Assign<kIsVolatile>(new_value.Ptr()); } template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags, @@ -843,13 +826,12 @@ inline bool Object::CasFieldWeakSequentiallyConsistentObjectWithoutWriteBarrier( if (kTransactionActive) { Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); } - HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); - HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); + uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - bool success = atomic_addr->CompareExchangeWeakSequentiallyConsistent(old_ref.reference_, - new_ref.reference_); + bool success = atomic_addr->CompareExchangeWeakSequentiallyConsistent(old_ref, new_ref); return success; } @@ -885,13 +867,12 @@ inline bool Object::CasFieldStrongSequentiallyConsistentObjectWithoutWriteBarrie if (kTransactionActive) { Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); } - HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); - HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); + uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - bool success = atomic_addr->CompareExchangeStrongSequentiallyConsistent(old_ref.reference_, - new_ref.reference_); + bool success = atomic_addr->CompareExchangeStrongSequentiallyConsistent(old_ref, new_ref); return success; } @@ -915,13 +896,12 @@ inline bool Object::CasFieldWeakRelaxedObjectWithoutWriteBarrier( if (kTransactionActive) { Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); } - HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); - HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); + uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - bool success = atomic_addr->CompareExchangeWeakRelaxed(old_ref.reference_, - new_ref.reference_); + bool success = atomic_addr->CompareExchangeWeakRelaxed(old_ref, new_ref); return success; } @@ -945,13 +925,12 @@ inline bool Object::CasFieldWeakReleaseObjectWithoutWriteBarrier( if (kTransactionActive) { Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); } - HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); - HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); + uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - bool success = atomic_addr->CompareExchangeWeakRelease(old_ref.reference_, - new_ref.reference_); + bool success = atomic_addr->CompareExchangeWeakRelease(old_ref, new_ref); return success; } diff --git a/runtime/mirror/object-readbarrier-inl.h b/runtime/mirror/object-readbarrier-inl.h index 69365af7fd..f0769409d4 100644 --- a/runtime/mirror/object-readbarrier-inl.h +++ b/runtime/mirror/object-readbarrier-inl.h @@ -211,13 +211,12 @@ inline bool Object::CasFieldStrongRelaxedObjectWithoutWriteBarrier( if (kTransactionActive) { Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); } - HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); - HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); + uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - bool success = atomic_addr->CompareExchangeStrongRelaxed(old_ref.reference_, - new_ref.reference_); + bool success = atomic_addr->CompareExchangeStrongRelaxed(old_ref, new_ref); return success; } @@ -241,13 +240,12 @@ inline bool Object::CasFieldStrongReleaseObjectWithoutWriteBarrier( if (kTransactionActive) { Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); } - HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); - HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value)); + uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - bool success = atomic_addr->CompareExchangeStrongRelease(old_ref.reference_, - new_ref.reference_); + bool success = atomic_addr->CompareExchangeStrongRelease(old_ref, new_ref); return success; } diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h index 886780f051..aedcd66082 100644 --- a/runtime/mirror/object.h +++ b/runtime/mirror/object.h @@ -380,7 +380,12 @@ class MANAGED LOCKABLE Object { template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, bool kIsVolatile = false> ALWAYS_INLINE uint8_t GetFieldBoolean(MemberOffset field_offset) - REQUIRES_SHARED(Locks::mutator_lock_); + REQUIRES_SHARED(Locks::mutator_lock_) { + if (kVerifyFlags & kVerifyThis) { + VerifyObject(this); + } + return GetField<uint8_t, kIsVolatile>(field_offset); + } template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, bool kIsVolatile = false> ALWAYS_INLINE int8_t GetFieldByte(MemberOffset field_offset) diff --git a/runtime/mirror/object_reference-inl.h b/runtime/mirror/object_reference-inl.h index 22fb83cb5c..60f3ce153f 100644 --- a/runtime/mirror/object_reference-inl.h +++ b/runtime/mirror/object_reference-inl.h @@ -30,17 +30,10 @@ void ObjectReference<kPoisonReferences, MirrorType>::Assign(ObjPtr<MirrorType> p } template<class MirrorType> -HeapReference<MirrorType> HeapReference<MirrorType>::FromObjPtr(ObjPtr<MirrorType> ptr) { - return HeapReference<MirrorType>(ptr.Ptr()); -} - -template<class MirrorType> bool HeapReference<MirrorType>::CasWeakRelaxed(MirrorType* expected_ptr, MirrorType* new_ptr) { - HeapReference<Object> expected_ref(HeapReference<Object>::FromMirrorPtr(expected_ptr)); - HeapReference<Object> new_ref(HeapReference<Object>::FromMirrorPtr(new_ptr)); - Atomic<uint32_t>* atomic_reference = reinterpret_cast<Atomic<uint32_t>*>(&this->reference_); - return atomic_reference->CompareExchangeWeakRelaxed(expected_ref.reference_, - new_ref.reference_); + return reference_.CompareExchangeWeakRelaxed( + Compression::Compress(expected_ptr), + Compression::Compress(new_ptr)); } } // namespace mirror diff --git a/runtime/mirror/object_reference.h b/runtime/mirror/object_reference.h index a96a120d68..c62ee6cb61 100644 --- a/runtime/mirror/object_reference.h +++ b/runtime/mirror/object_reference.h @@ -17,6 +17,7 @@ #ifndef ART_RUNTIME_MIRROR_OBJECT_REFERENCE_H_ #define ART_RUNTIME_MIRROR_OBJECT_REFERENCE_H_ +#include "atomic.h" #include "base/mutex.h" // For Locks::mutator_lock_. #include "globals.h" #include "obj_ptr.h" @@ -30,20 +31,43 @@ class Object; // extra platform specific padding. #define MANAGED PACKED(4) +template<bool kPoisonReferences, class MirrorType> +class PtrCompression { + public: + // Compress reference to its bit representation. + static uint32_t Compress(MirrorType* mirror_ptr) { + uintptr_t as_bits = reinterpret_cast<uintptr_t>(mirror_ptr); + return static_cast<uint32_t>(kPoisonReferences ? -as_bits : as_bits); + } + + // Uncompress an encoded reference from its bit representation. + static MirrorType* Decompress(uint32_t ref) { + uintptr_t as_bits = kPoisonReferences ? -ref : ref; + return reinterpret_cast<MirrorType*>(as_bits); + } + + // Convert an ObjPtr to a compressed reference. + static uint32_t Compress(ObjPtr<MirrorType> ptr) REQUIRES_SHARED(Locks::mutator_lock_) { + return Compress(ptr.Ptr()); + } +}; + // Value type representing a reference to a mirror::Object of type MirrorType. template<bool kPoisonReferences, class MirrorType> class MANAGED ObjectReference { + private: + using Compression = PtrCompression<kPoisonReferences, MirrorType>; + public: - MirrorType* AsMirrorPtr() const REQUIRES_SHARED(Locks::mutator_lock_) { - return UnCompress(); + MirrorType* AsMirrorPtr() const { + return Compression::Decompress(reference_); } - void Assign(MirrorType* other) REQUIRES_SHARED(Locks::mutator_lock_) { - reference_ = Compress(other); + void Assign(MirrorType* other) { + reference_ = Compression::Compress(other); } - void Assign(ObjPtr<MirrorType> ptr) - REQUIRES_SHARED(Locks::mutator_lock_); + void Assign(ObjPtr<MirrorType> ptr) REQUIRES_SHARED(Locks::mutator_lock_); void Clear() { reference_ = 0; @@ -58,48 +82,71 @@ class MANAGED ObjectReference { return reference_; } - protected: - explicit ObjectReference(MirrorType* mirror_ptr) - REQUIRES_SHARED(Locks::mutator_lock_) - : reference_(Compress(mirror_ptr)) { - } - - // Compress reference to its bit representation. - static uint32_t Compress(MirrorType* mirror_ptr) REQUIRES_SHARED(Locks::mutator_lock_) { - uintptr_t as_bits = reinterpret_cast<uintptr_t>(mirror_ptr); - return static_cast<uint32_t>(kPoisonReferences ? -as_bits : as_bits); + static ObjectReference<kPoisonReferences, MirrorType> FromMirrorPtr(MirrorType* mirror_ptr) + REQUIRES_SHARED(Locks::mutator_lock_) { + return ObjectReference<kPoisonReferences, MirrorType>(mirror_ptr); } - // Uncompress an encoded reference from its bit representation. - MirrorType* UnCompress() const REQUIRES_SHARED(Locks::mutator_lock_) { - uintptr_t as_bits = kPoisonReferences ? -reference_ : reference_; - return reinterpret_cast<MirrorType*>(as_bits); + protected: + explicit ObjectReference(MirrorType* mirror_ptr) REQUIRES_SHARED(Locks::mutator_lock_) + : reference_(Compression::Compress(mirror_ptr)) { } - friend class Object; - // The encoded reference to a mirror::Object. uint32_t reference_; }; // References between objects within the managed heap. +// Similar API to ObjectReference, but not a value type. Supports atomic access. template<class MirrorType> -class MANAGED HeapReference : public ObjectReference<kPoisonHeapReferences, MirrorType> { +class MANAGED HeapReference { + private: + using Compression = PtrCompression<kPoisonHeapReferences, MirrorType>; + public: + HeapReference() REQUIRES_SHARED(Locks::mutator_lock_) : HeapReference(nullptr) {} + + template <bool kIsVolatile = false> + MirrorType* AsMirrorPtr() const REQUIRES_SHARED(Locks::mutator_lock_) { + return Compression::Decompress( + kIsVolatile ? reference_.LoadSequentiallyConsistent() : reference_.LoadJavaData()); + } + + template <bool kIsVolatile = false> + void Assign(MirrorType* other) REQUIRES_SHARED(Locks::mutator_lock_) { + if (kIsVolatile) { + reference_.StoreSequentiallyConsistent(Compression::Compress(other)); + } else { + reference_.StoreJavaData(Compression::Compress(other)); + } + } + + template <bool kIsVolatile = false> + void Assign(ObjPtr<MirrorType> ptr) REQUIRES_SHARED(Locks::mutator_lock_); + + void Clear() { + reference_.StoreJavaData(0); + DCHECK(IsNull()); + } + + bool IsNull() const { + return reference_.LoadJavaData() == 0; + } + static HeapReference<MirrorType> FromMirrorPtr(MirrorType* mirror_ptr) REQUIRES_SHARED(Locks::mutator_lock_) { return HeapReference<MirrorType>(mirror_ptr); } - static HeapReference<MirrorType> FromObjPtr(ObjPtr<MirrorType> ptr) - REQUIRES_SHARED(Locks::mutator_lock_); - bool CasWeakRelaxed(MirrorType* old_ptr, MirrorType* new_ptr) REQUIRES_SHARED(Locks::mutator_lock_); private: explicit HeapReference(MirrorType* mirror_ptr) REQUIRES_SHARED(Locks::mutator_lock_) - : ObjectReference<kPoisonHeapReferences, MirrorType>(mirror_ptr) {} + : reference_(Compression::Compress(mirror_ptr)) {} + + // The encoded reference to a mirror::Object. Atomically updateable. + Atomic<uint32_t> reference_; }; static_assert(sizeof(mirror::HeapReference<mirror::Object>) == kHeapReferenceSize, diff --git a/runtime/native/java_lang_reflect_Field.cc b/runtime/native/java_lang_reflect_Field.cc index 9f59a1f751..2e4dd8a599 100644 --- a/runtime/native/java_lang_reflect_Field.cc +++ b/runtime/native/java_lang_reflect_Field.cc @@ -27,7 +27,7 @@ #include "dex_file_annotations.h" #include "jni_internal.h" #include "mirror/class-inl.h" -#include "mirror/field.h" +#include "mirror/field-inl.h" #include "native_util.h" #include "reflection-inl.h" #include "scoped_fast_native_object_access-inl.h" diff --git a/runtime/native/sun_misc_Unsafe.cc b/runtime/native/sun_misc_Unsafe.cc index 761362fcb2..b2bdeed5cb 100644 --- a/runtime/native/sun_misc_Unsafe.cc +++ b/runtime/native/sun_misc_Unsafe.cc @@ -67,10 +67,12 @@ static jboolean Unsafe_compareAndSwapObject(JNIEnv* env, jobject, jobject javaOb if (kUseReadBarrier) { // Need to make sure the reference stored in the field is a to-space one before attempting the // CAS or the CAS could fail incorrectly. + // Note that the read barrier load does NOT need to be volatile. mirror::HeapReference<mirror::Object>* field_addr = reinterpret_cast<mirror::HeapReference<mirror::Object>*>( reinterpret_cast<uint8_t*>(obj.Ptr()) + static_cast<size_t>(offset)); - ReadBarrier::Barrier<mirror::Object, kWithReadBarrier, /* kAlwaysUpdateField */ true>( + ReadBarrier::Barrier<mirror::Object, /* kIsVolatile */ false, kWithReadBarrier, + /* kAlwaysUpdateField */ true>( obj.Ptr(), MemberOffset(offset), field_addr); @@ -112,6 +114,7 @@ static void Unsafe_putOrderedInt(JNIEnv* env, jobject, jobject javaObj, jlong of jint newValue) { ScopedFastNativeObjectAccess soa(env); ObjPtr<mirror::Object> obj = soa.Decode<mirror::Object>(javaObj); + // TODO: A release store is likely to be faster on future processors. QuasiAtomic::ThreadFenceRelease(); // JNI must use non transactional mode. obj->SetField32<false>(MemberOffset(offset), newValue); diff --git a/runtime/quick_exception_handler.cc b/runtime/quick_exception_handler.cc index db10103c4b..b592247da3 100644 --- a/runtime/quick_exception_handler.cc +++ b/runtime/quick_exception_handler.cc @@ -533,21 +533,19 @@ void QuickExceptionHandler::DeoptimizeStack() { void QuickExceptionHandler::DeoptimizeSingleFrame(DeoptimizationKind kind) { DCHECK(is_deoptimization_); - if (VLOG_IS_ON(deopt) || kDebugExceptionDelivery) { - LOG(INFO) << "Single-frame deopting:"; - DumpFramesWithType(self_, true); - } - DeoptimizeStackVisitor visitor(self_, context_, this, true); visitor.WalkStack(true); // Compiled code made an explicit deoptimization. ArtMethod* deopt_method = visitor.GetSingleFrameDeoptMethod(); DCHECK(deopt_method != nullptr); - LOG(INFO) << "Deoptimizing " - << deopt_method->PrettyMethod() - << " due to " - << GetDeoptimizationKindName(kind); + if (VLOG_IS_ON(deopt) || kDebugExceptionDelivery) { + LOG(INFO) << "Single-frame deopting: " + << deopt_method->PrettyMethod() + << " due to " + << GetDeoptimizationKindName(kind); + DumpFramesWithType(self_, /* details */ true); + } if (Runtime::Current()->UseJitCompilation()) { Runtime::Current()->GetJit()->GetCodeCache()->InvalidateCompiledCodeFor( deopt_method, visitor.GetSingleFrameDeoptQuickMethodHeader()); diff --git a/runtime/read_barrier-inl.h b/runtime/read_barrier-inl.h index b0935c0b56..642459924e 100644 --- a/runtime/read_barrier-inl.h +++ b/runtime/read_barrier-inl.h @@ -33,7 +33,8 @@ namespace art { // Disabled for performance reasons. static constexpr bool kCheckDebugDisallowReadBarrierCount = false; -template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kAlwaysUpdateField> +template <typename MirrorType, bool kIsVolatile, ReadBarrierOption kReadBarrierOption, + bool kAlwaysUpdateField> inline MirrorType* ReadBarrier::Barrier( mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr) { constexpr bool with_read_barrier = kReadBarrierOption == kWithReadBarrier; @@ -55,7 +56,7 @@ inline MirrorType* ReadBarrier::Barrier( } ref_addr = reinterpret_cast<mirror::HeapReference<MirrorType>*>( fake_address_dependency | reinterpret_cast<uintptr_t>(ref_addr)); - MirrorType* ref = ref_addr->AsMirrorPtr(); + MirrorType* ref = ref_addr->template AsMirrorPtr<kIsVolatile>(); MirrorType* old_ref = ref; if (is_gray) { // Slow-path. @@ -71,9 +72,9 @@ inline MirrorType* ReadBarrier::Barrier( return ref; } else if (kUseBrooksReadBarrier) { // To be implemented. - return ref_addr->AsMirrorPtr(); + return ref_addr->template AsMirrorPtr<kIsVolatile>(); } else if (kUseTableLookupReadBarrier) { - MirrorType* ref = ref_addr->AsMirrorPtr(); + MirrorType* ref = ref_addr->template AsMirrorPtr<kIsVolatile>(); MirrorType* old_ref = ref; // The heap or the collector can be null at startup. TODO: avoid the need for this null check. gc::Heap* heap = Runtime::Current()->GetHeap(); @@ -93,7 +94,7 @@ inline MirrorType* ReadBarrier::Barrier( } } else { // No read barrier. - return ref_addr->AsMirrorPtr(); + return ref_addr->template AsMirrorPtr<kIsVolatile>(); } } diff --git a/runtime/read_barrier.h b/runtime/read_barrier.h index d36acbcdc4..8a106aabb0 100644 --- a/runtime/read_barrier.h +++ b/runtime/read_barrier.h @@ -46,9 +46,13 @@ class ReadBarrier { // fast-debug environment. DECLARE_RUNTIME_DEBUG_FLAG(kEnableReadBarrierInvariantChecks); + // Return the reference at ref_addr, invoking read barrier as appropriate. + // Ref_addr is an address within obj. // It's up to the implementation whether the given field gets updated whereas the return value // must be an updated reference unless kAlwaysUpdateField is true. - template <typename MirrorType, ReadBarrierOption kReadBarrierOption = kWithReadBarrier, + template <typename MirrorType, + bool kIsVolatile, + ReadBarrierOption kReadBarrierOption = kWithReadBarrier, bool kAlwaysUpdateField = false> ALWAYS_INLINE static MirrorType* Barrier( mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr) diff --git a/runtime/runtime-inl.h b/runtime/runtime-inl.h index 609f0d658d..4584351a6a 100644 --- a/runtime/runtime-inl.h +++ b/runtime/runtime-inl.h @@ -49,7 +49,9 @@ inline QuickMethodFrameInfo Runtime::GetRuntimeMethodFrameInfo(ArtMethod* method } else if (method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveRefsOnly)) { return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveRefsOnly); } else { - DCHECK_EQ(method, GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverything)); + DCHECK(method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverything) || + method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverythingForClinit) || + method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverythingForSuspendCheck)); return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveEverything); } } diff --git a/runtime/runtime.h b/runtime/runtime.h index 0c1344e11a..4e84468744 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -714,7 +714,7 @@ class Runtime { static constexpr int kProfileForground = 0; static constexpr int kProfileBackground = 1; - static constexpr uint32_t kCalleeSaveSize = 4u; + static constexpr uint32_t kCalleeSaveSize = 6u; // 64 bit so that we can share the same asm offsets for both 32 and 64 bits. uint64_t callee_save_methods_[kCalleeSaveSize]; diff --git a/runtime/stack.cc b/runtime/stack.cc index 19df0d26a1..2e0653666f 100644 --- a/runtime/stack.cc +++ b/runtime/stack.cc @@ -634,7 +634,7 @@ static void AssertPcIsWithinQuickCode(ArtMethod* method, uintptr_t pc) void StackVisitor::SanityCheckFrame() const { if (kIsDebugBuild) { ArtMethod* method = GetMethod(); - auto* declaring_class = method->GetDeclaringClass(); + mirror::Class* declaring_class = method->GetDeclaringClass(); // Runtime methods have null declaring class. if (!method->IsRuntimeMethod()) { CHECK(declaring_class != nullptr); @@ -647,11 +647,14 @@ void StackVisitor::SanityCheckFrame() const { LinearAlloc* const linear_alloc = runtime->GetLinearAlloc(); if (!linear_alloc->Contains(method)) { // Check class linker linear allocs. - mirror::Class* klass = method->GetDeclaringClass(); + // We get the canonical method as copied methods may have their declaring + // class from another class loader. + ArtMethod* canonical = method->GetCanonicalMethod(); + mirror::Class* klass = canonical->GetDeclaringClass(); LinearAlloc* const class_linear_alloc = (klass != nullptr) ? runtime->GetClassLinker()->GetAllocatorForClassLoader(klass->GetClassLoader()) : linear_alloc; - if (!class_linear_alloc->Contains(method)) { + if (!class_linear_alloc->Contains(canonical)) { // Check image space. bool in_image = false; for (auto& space : runtime->GetHeap()->GetContinuousSpaces()) { @@ -660,14 +663,14 @@ void StackVisitor::SanityCheckFrame() const { const auto& header = image_space->GetImageHeader(); const ImageSection& methods = header.GetMethodsSection(); const ImageSection& runtime_methods = header.GetRuntimeMethodsSection(); - const size_t offset = reinterpret_cast<const uint8_t*>(method) - image_space->Begin(); + const size_t offset = reinterpret_cast<const uint8_t*>(canonical) - image_space->Begin(); if (methods.Contains(offset) || runtime_methods.Contains(offset)) { in_image = true; break; } } } - CHECK(in_image) << method->PrettyMethod() << " not in linear alloc or image"; + CHECK(in_image) << canonical->PrettyMethod() << " not in linear alloc or image"; } } if (cur_quick_frame_ != nullptr) { diff --git a/runtime/thread.h b/runtime/thread.h index 776096a7bb..7540fd2563 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -1705,8 +1705,13 @@ class Thread { class SCOPED_CAPABILITY ScopedAssertNoThreadSuspension { public: - ALWAYS_INLINE explicit ScopedAssertNoThreadSuspension(const char* cause) - ACQUIRE(Roles::uninterruptible_) { + ALWAYS_INLINE ScopedAssertNoThreadSuspension(const char* cause, + bool enabled = true) + ACQUIRE(Roles::uninterruptible_) + : enabled_(enabled) { + if (!enabled_) { + return; + } if (kIsDebugBuild) { self_ = Thread::Current(); old_cause_ = self_->StartAssertNoThreadSuspension(cause); @@ -1715,6 +1720,9 @@ class SCOPED_CAPABILITY ScopedAssertNoThreadSuspension { } } ALWAYS_INLINE ~ScopedAssertNoThreadSuspension() RELEASE(Roles::uninterruptible_) { + if (!enabled_) { + return; + } if (kIsDebugBuild) { self_->EndAssertNoThreadSuspension(old_cause_); } else { @@ -1724,6 +1732,7 @@ class SCOPED_CAPABILITY ScopedAssertNoThreadSuspension { private: Thread* self_; + const bool enabled_; const char* old_cause_; }; diff --git a/runtime/vdex_file.cc b/runtime/vdex_file.cc index e8f947c95a..b95522062e 100644 --- a/runtime/vdex_file.cc +++ b/runtime/vdex_file.cc @@ -20,6 +20,7 @@ #include <memory> +#include "base/bit_utils.h" #include "base/logging.h" #include "base/stl_util.h" #include "base/unix_file/fd_file.h" @@ -134,6 +135,9 @@ const uint8_t* VdexFile::GetNextDexFileData(const uint8_t* cursor) const { } else { // Fetch the next dex file. Return null if there is none. const uint8_t* data = cursor + reinterpret_cast<const DexFile::Header*>(cursor)->file_size_; + // Dex files are required to be 4 byte aligned. the OatWriter makes sure they are, see + // OatWriter::SeekToDexFiles. + data = AlignUp(data, 4); return (data == DexEnd()) ? nullptr : data; } } diff --git a/test/1338-gc-no-los/expected.txt b/test/1338-gc-no-los/expected.txt new file mode 100644 index 0000000000..36bec43b8b --- /dev/null +++ b/test/1338-gc-no-los/expected.txt @@ -0,0 +1 @@ +131072 200 true diff --git a/test/1338-gc-no-los/info.txt b/test/1338-gc-no-los/info.txt new file mode 100644 index 0000000000..2e27357b80 --- /dev/null +++ b/test/1338-gc-no-los/info.txt @@ -0,0 +1 @@ +Test that the GC works with no large object space. Regression test for b/64393515.
\ No newline at end of file diff --git a/test/1338-gc-no-los/run b/test/1338-gc-no-los/run new file mode 100755 index 0000000000..f3c43fbce6 --- /dev/null +++ b/test/1338-gc-no-los/run @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +./default-run "$@" --runtime-option -XX:LargeObjectSpace=disabled diff --git a/test/1338-gc-no-los/src-art/Main.java b/test/1338-gc-no-los/src-art/Main.java new file mode 100644 index 0000000000..662fa7e610 --- /dev/null +++ b/test/1338-gc-no-los/src-art/Main.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import dalvik.system.VMRuntime; +import java.lang.ref.WeakReference; +import java.lang.reflect.Method; + +public class Main { + public static void main(String[] args) { + try { + // Allocate a large object. + byte[] arr = new byte[128 * 1024]; + // Allocate a non movable object. + byte[] arr2 = (byte[])VMRuntime.getRuntime().newNonMovableArray(Byte.TYPE, 200); + // Put the array in a weak reference so that IsMarked is called by the GC. + WeakReference weakRef = new WeakReference(arr2); + // Do a GC. + Runtime.getRuntime().gc(); + arr[0] = 1; + arr2[0] = 1; + System.out.println(arr.length + " " + arr2.length + " " + (weakRef.get() != null)); + } catch (Exception e) { + System.out.println(e); + } + } +} diff --git a/test/449-checker-bce/src/Main.java b/test/449-checker-bce/src/Main.java index 510354073a..f466eea97b 100644 --- a/test/449-checker-bce/src/Main.java +++ b/test/449-checker-bce/src/Main.java @@ -876,6 +876,91 @@ public class Main { return true; } + /// CHECK-START: void Main.modArrayIndex1(int[]) BCE (before) + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + + /// CHECK-START: void Main.modArrayIndex1(int[]) BCE (after) + /// CHECK-NOT: Deoptimize + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-NOT: BoundsCheck + /// CHECK-DAG: ArraySet + public static void modArrayIndex1(int[] array) { + for(int i = 0; i < 100; i++) { + // Cannot statically eliminate, for example, when array.length == 5. + // Currently dynamic BCE isn't applied for this case. + array[i % 10] = i; + // Can be eliminated by BCE. + array[i % array.length] = i; + } + } + + /// CHECK-START: void Main.modArrayIndex2(int[], int) BCE (before) + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + + /// CHECK-START: void Main.modArrayIndex2(int[], int) BCE (after) + /// CHECK-NOT: Deoptimize + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + public static void modArrayIndex2(int array[], int index) { + for(int i = 0; i < 100; i++) { + // Both bounds checks cannot be statically eliminated, because index can be < 0. + // Currently dynamic BCE isn't applied for this case. + array[(index+i) % 10] = i; + array[(index+i) % array.length] = i; + } + } + + static final int[] staticArray = new int[10]; + + /// CHECK-START: void Main.modArrayIndex3() BCE (before) + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + + /// CHECK-START: void Main.modArrayIndex3() BCE (after) + /// CHECK-NOT: Deoptimize + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-NOT: BoundsCheck + /// CHECK-DAG: ArraySet + public static void modArrayIndex3() { + for(int i = 0; i < 100; i++) { + // Currently dynamic BCE isn't applied for this case. + staticArray[i % 10] = i; + // Can be eliminated by BCE. + staticArray[i % staticArray.length] = i; + } + } + + /// CHECK-START: void Main.modArrayIndex4() BCE (before) + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-DAG: BoundsCheck + /// CHECK-DAG: ArraySet + + /// CHECK-START: void Main.modArrayIndex4() BCE (after) + /// CHECK-NOT: BoundsCheck + /// CHECK-DAG: ArraySet + /// CHECK-NOT: BoundsCheck + /// CHECK-DAG: ArraySet + public static void modArrayIndex4() { + int[] array = new int[20]; + for(int i = 0; i < 100; i++) { + // The local array length is statically know. Both can be eliminated by BCE. + array[i % 10] = i; + array[i % array.length] = i; + } + } /// CHECK-START: void Main.bubbleSort(int[]) GVN (before) /// CHECK: BoundsCheck diff --git a/test/623-checker-loop-regressions/src/Main.java b/test/623-checker-loop-regressions/src/Main.java index fc7bcb2f8f..0e8561299c 100644 --- a/test/623-checker-loop-regressions/src/Main.java +++ b/test/623-checker-loop-regressions/src/Main.java @@ -441,6 +441,29 @@ public class Main { } } + /// CHECK-START: int Main.feedsIntoDeopt(int[]) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop2:B\d+>> outer_loop:none + // + /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" + // + /// CHECK-START: int Main.feedsIntoDeopt(int[]) loop_optimization (after) + /// CHECK-DAG: Phi loop:{{B\d+}} outer_loop:none + /// CHECK-NOT: Phi + static int feedsIntoDeopt(int[] a) { + // Reduction should be removed. + int r = 0; + for (int i = 0; i < 100; i++) { + r += 10; + } + // Even though uses feed into deopts of BCE. + for (int i = 1; i < 100; i++) { + a[i] = a[i - 1]; + } + return r; + } + public static void main(String[] args) { expectEquals(10, earlyExitFirst(-1)); for (int i = 0; i <= 10; i++) { @@ -556,6 +579,13 @@ public class Main { inductionMax(yy); + int[] f = new int[100]; + f[0] = 11; + expectEquals(1000, feedsIntoDeopt(f)); + for (int i = 0; i < 100; i++) { + expectEquals(11, f[i]); + } + System.out.println("passed"); } diff --git a/test/661-checker-simd-reduc/expected.txt b/test/661-checker-simd-reduc/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/661-checker-simd-reduc/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/661-checker-simd-reduc/info.txt b/test/661-checker-simd-reduc/info.txt new file mode 100644 index 0000000000..4d071fb03a --- /dev/null +++ b/test/661-checker-simd-reduc/info.txt @@ -0,0 +1 @@ +Functional tests on vectorization of the most basic reductions. diff --git a/test/661-checker-simd-reduc/src/Main.java b/test/661-checker-simd-reduc/src/Main.java new file mode 100644 index 0000000000..741b5fa9a5 --- /dev/null +++ b/test/661-checker-simd-reduc/src/Main.java @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Tests for simple integral reductions: same type for accumulator and data. + */ +public class Main { + + static final int N = 500; + + // + // Basic reductions in loops. + // + + // TODO: vectorize these (second step of b/64091002 plan) + + private static byte reductionByte(byte[] x) { + byte sum = 0; + for (int i = 0; i < x.length; i++) { + sum += x[i]; + } + return sum; + } + + private static short reductionShort(short[] x) { + short sum = 0; + for (int i = 0; i < x.length; i++) { + sum += x[i]; + } + return sum; + } + + private static char reductionChar(char[] x) { + char sum = 0; + for (int i = 0; i < x.length; i++) { + sum += x[i]; + } + return sum; + } + + private static int reductionInt(int[] x) { + int sum = 0; + for (int i = 0; i < x.length; i++) { + sum += x[i]; + } + return sum; + } + + private static long reductionLong(long[] x) { + long sum = 0; + for (int i = 0; i < x.length; i++) { + sum += x[i]; + } + return sum; + } + + private static byte reductionMinusByte(byte[] x) { + byte sum = 0; + for (int i = 0; i < x.length; i++) { + sum -= x[i]; + } + return sum; + } + + private static short reductionMinusShort(short[] x) { + short sum = 0; + for (int i = 0; i < x.length; i++) { + sum -= x[i]; + } + return sum; + } + + private static char reductionMinusChar(char[] x) { + char sum = 0; + for (int i = 0; i < x.length; i++) { + sum -= x[i]; + } + return sum; + } + + private static int reductionMinusInt(int[] x) { + int sum = 0; + for (int i = 0; i < x.length; i++) { + sum -= x[i]; + } + return sum; + } + + private static long reductionMinusLong(long[] x) { + long sum = 0; + for (int i = 0; i < x.length; i++) { + sum -= x[i]; + } + return sum; + } + + private static byte reductionMinByte(byte[] x) { + byte min = Byte.MAX_VALUE; + for (int i = 0; i < x.length; i++) { + min = (byte) Math.min(min, x[i]); + } + return min; + } + + private static short reductionMinShort(short[] x) { + short min = Short.MAX_VALUE; + for (int i = 0; i < x.length; i++) { + min = (short) Math.min(min, x[i]); + } + return min; + } + + private static char reductionMinChar(char[] x) { + char min = Character.MAX_VALUE; + for (int i = 0; i < x.length; i++) { + min = (char) Math.min(min, x[i]); + } + return min; + } + + private static int reductionMinInt(int[] x) { + int min = Integer.MAX_VALUE; + for (int i = 0; i < x.length; i++) { + min = Math.min(min, x[i]); + } + return min; + } + + private static long reductionMinLong(long[] x) { + long min = Long.MAX_VALUE; + for (int i = 0; i < x.length; i++) { + min = Math.min(min, x[i]); + } + return min; + } + + private static byte reductionMaxByte(byte[] x) { + byte max = Byte.MIN_VALUE; + for (int i = 0; i < x.length; i++) { + max = (byte) Math.max(max, x[i]); + } + return max; + } + + private static short reductionMaxShort(short[] x) { + short max = Short.MIN_VALUE; + for (int i = 0; i < x.length; i++) { + max = (short) Math.max(max, x[i]); + } + return max; + } + + private static char reductionMaxChar(char[] x) { + char max = Character.MIN_VALUE; + for (int i = 0; i < x.length; i++) { + max = (char) Math.max(max, x[i]); + } + return max; + } + + private static int reductionMaxInt(int[] x) { + int max = Integer.MIN_VALUE; + for (int i = 0; i < x.length; i++) { + max = Math.max(max, x[i]); + } + return max; + } + + private static long reductionMaxLong(long[] x) { + long max = Long.MIN_VALUE; + for (int i = 0; i < x.length; i++) { + max = Math.max(max, x[i]); + } + return max; + } + + // + // A few special cases. + // + + // TODO: consider unrolling + + private static int reductionInt10(int[] x) { + int sum = 0; + // Amenable to complete unrolling. + for (int i = 10; i <= 10; i++) { + sum += x[i]; + } + return sum; + } + + private static int reductionMinusInt10(int[] x) { + int sum = 0; + // Amenable to complete unrolling. + for (int i = 10; i <= 10; i++) { + sum -= x[i]; + } + return sum; + } + + private static int reductionMinInt10(int[] x) { + int min = Integer.MAX_VALUE; + // Amenable to complete unrolling. + for (int i = 10; i <= 10; i++) { + min = Math.min(min, x[i]); + } + return min; + } + + private static int reductionMaxInt10(int[] x) { + int max = Integer.MIN_VALUE; + // Amenable to complete unrolling. + for (int i = 10; i <= 10; i++) { + max = Math.max(max, x[i]); + } + return max; + } + + // + // Main driver. + // + + public static void main(String[] args) { + byte[] xb = new byte[N]; + short[] xs = new short[N]; + char[] xc = new char[N]; + int[] xi = new int[N]; + long[] xl = new long[N]; + for (int i = 0, k = -17; i < N; i++, k += 3) { + xb[i] = (byte) k; + xs[i] = (short) k; + xc[i] = (char) k; + xi[i] = k; + xl[i] = k; + } + + // Test various reductions in loops. + expectEquals(-74, reductionByte(xb)); + expectEquals(-27466, reductionShort(xs)); + expectEquals(38070, reductionChar(xc)); + expectEquals(365750, reductionInt(xi)); + expectEquals(365750L, reductionLong(xl)); + expectEquals(74, reductionMinusByte(xb)); + expectEquals(27466, reductionMinusShort(xs)); + expectEquals(27466, reductionMinusChar(xc)); + expectEquals(-365750, reductionMinusInt(xi)); + expectEquals(-365750L, reductionMinusLong(xl)); + expectEquals(-128, reductionMinByte(xb)); + expectEquals(-17, reductionMinShort(xs)); + expectEquals(1, reductionMinChar(xc)); + expectEquals(-17, reductionMinInt(xi)); + expectEquals(-17L, reductionMinLong(xl)); + expectEquals(127, reductionMaxByte(xb)); + expectEquals(1480, reductionMaxShort(xs)); + expectEquals(65534, reductionMaxChar(xc)); + expectEquals(1480, reductionMaxInt(xi)); + expectEquals(1480L, reductionMaxLong(xl)); + + // Test special cases. + expectEquals(13, reductionInt10(xi)); + expectEquals(-13, reductionMinusInt10(xi)); + expectEquals(13, reductionMinInt10(xi)); + expectEquals(13, reductionMaxInt10(xi)); + + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } + + private static void expectEquals(long expected, long result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } +} diff --git a/test/661-classloader-allocator/expected.txt b/test/661-classloader-allocator/expected.txt new file mode 100644 index 0000000000..6a5618ebc6 --- /dev/null +++ b/test/661-classloader-allocator/expected.txt @@ -0,0 +1 @@ +JNI_OnLoad called diff --git a/test/661-classloader-allocator/info.txt b/test/661-classloader-allocator/info.txt new file mode 100644 index 0000000000..872b5e0a5a --- /dev/null +++ b/test/661-classloader-allocator/info.txt @@ -0,0 +1,3 @@ +Regression test for copied methods which used to not +be (re-)allocated with the right class loader allocator, +which crashed the JIT profiler. diff --git a/test/661-classloader-allocator/src-ex/OtherClass.java b/test/661-classloader-allocator/src-ex/OtherClass.java new file mode 100644 index 0000000000..e59cb953fc --- /dev/null +++ b/test/661-classloader-allocator/src-ex/OtherClass.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p1; + +interface Itf { + public default int defaultMethod() { + return 42; + } +} + +public class OtherClass implements Itf { + public void foo() { + } +} diff --git a/test/661-classloader-allocator/src/Main.java b/test/661-classloader-allocator/src/Main.java new file mode 100644 index 0000000000..40f8f7a4bf --- /dev/null +++ b/test/661-classloader-allocator/src/Main.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.lang.reflect.Constructor; + +public class Main { + static final String DEX_FILE = + System.getenv("DEX_LOCATION") + "/661-classloader-allocator-ex.jar"; + static final String LIBRARY_SEARCH_PATH = System.getProperty("java.library.path"); + + private static void doUnloading() { + // Stop the JIT to ensure its threads and work queue are not keeping classes + // artifically alive. + stopJit(); + // Do multiple GCs to prevent rare flakiness if some other thread is keeping the + // classloader live. + for (int i = 0; i < 5; ++i) { + Runtime.getRuntime().gc(); + } + startJit(); + } + + public static void main(String[] args) throws Exception { + System.loadLibrary(args[0]); + loadClass(); + doUnloading(); + // fetchProfiles iterate over the ProfilingInfo, we used to crash in the presence + // of unloaded copied methods. + fetchProfiles(); + } + + public static void loadClass() throws Exception { + Class<?> pathClassLoader = Class.forName("dalvik.system.PathClassLoader"); + if (pathClassLoader == null) { + throw new AssertionError("Couldn't find path class loader class"); + } + Constructor<?> constructor = + pathClassLoader.getDeclaredConstructor(String.class, String.class, ClassLoader.class); + ClassLoader loader = (ClassLoader) constructor.newInstance( + DEX_FILE, LIBRARY_SEARCH_PATH, ClassLoader.getSystemClassLoader()); + Class<?> otherClass = loader.loadClass("p1.OtherClass"); + ensureJitCompiled(otherClass, "foo"); + } + + public static native void ensureJitCompiled(Class<?> cls, String methodName); + public static native void fetchProfiles(); + public static native void stopJit(); + public static native void startJit(); +} diff --git a/test/662-regression-alias/expected.txt b/test/662-regression-alias/expected.txt new file mode 100644 index 0000000000..7250211df8 --- /dev/null +++ b/test/662-regression-alias/expected.txt @@ -0,0 +1 @@ +passed 127 4 diff --git a/test/662-regression-alias/info.txt b/test/662-regression-alias/info.txt new file mode 100644 index 0000000000..584d3ee3d3 --- /dev/null +++ b/test/662-regression-alias/info.txt @@ -0,0 +1 @@ +Regression test on missed array element aliasing. diff --git a/test/662-regression-alias/src/Main.java b/test/662-regression-alias/src/Main.java new file mode 100644 index 0000000000..aad61e254d --- /dev/null +++ b/test/662-regression-alias/src/Main.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Regression test on ARM-scheduling/array-aliasing bug (b/64018485). + */ +public class Main { + + // + // Mimic original bug. + // + + static void setFields(int[] fields) { + if (fields == null || fields.length < 6) + fields = new int[6]; // creates phi + fields[5] = 127; + } + + static void processFieldValues(int field0, int field1, int field2, + int field3, int field4, int field5) { + if (field5 != 127) { + throw new Error("field = " + field5); + } else if (field0 != 0) { + processFieldValues(0, 0, 0, 0, 0, 0); // disable inlining + } + } + + static int doit(int pass) { + int[] fields = new int[6]; + for (; ; pass++) { + setFields(fields); + processFieldValues(fields[0], fields[1], fields[2], + fields[3], fields[4], fields[5]); + if (pass == 0) + break; + } + return fields[5]; + } + + // + // Similar situation. + // + + private static int aliasing(boolean f) { + int[] array = new int[6]; + int[] array2 = null; + int s = 0; + for (int i = 0; i < 1; i++) { + if (f) { + array2 = array; + } + array2[1] = 4; + s = array[1]; + } + return s; + } + + // + // Main driver. + // + + static public void main(String[] args) { + int r = doit(0); + int s = aliasing(true); + System.out.println("passed " + r + " " + s); + } +} diff --git a/test/663-odd-dex-size/classes.dex b/test/663-odd-dex-size/classes.dex Binary files differnew file mode 100644 index 0000000000..633e3a2d70 --- /dev/null +++ b/test/663-odd-dex-size/classes.dex diff --git a/test/663-odd-dex-size/expected.txt b/test/663-odd-dex-size/expected.txt new file mode 100644 index 0000000000..3da1ec26e9 --- /dev/null +++ b/test/663-odd-dex-size/expected.txt @@ -0,0 +1 @@ +HelloWorld diff --git a/test/663-odd-dex-size/info.txt b/test/663-odd-dex-size/info.txt new file mode 100644 index 0000000000..11a50e039e --- /dev/null +++ b/test/663-odd-dex-size/info.txt @@ -0,0 +1,14 @@ +Test for a dex file with an odd size in a vdex file. + +The code in the file is: + +class Main { + public static void main(String[] args) { + System.out.println("HelloWorld"); + } +} + +The generated dex file was then manually edited to: +1) Add 1 to the size value in the dex header. +2) Add 1 byte to the file. +3) Change the checksum in the dex header. diff --git a/test/663-odd-dex-size2/663-odd-dex-size2.jar b/test/663-odd-dex-size2/663-odd-dex-size2.jar Binary files differnew file mode 100644 index 0000000000..a29224e503 --- /dev/null +++ b/test/663-odd-dex-size2/663-odd-dex-size2.jar diff --git a/test/663-odd-dex-size2/build b/test/663-odd-dex-size2/build new file mode 100644 index 0000000000..5636558dad --- /dev/null +++ b/test/663-odd-dex-size2/build @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Nothing to do diff --git a/test/663-odd-dex-size2/expected.txt b/test/663-odd-dex-size2/expected.txt new file mode 100644 index 0000000000..3da1ec26e9 --- /dev/null +++ b/test/663-odd-dex-size2/expected.txt @@ -0,0 +1 @@ +HelloWorld diff --git a/test/663-odd-dex-size2/info.txt b/test/663-odd-dex-size2/info.txt new file mode 100644 index 0000000000..900394d74a --- /dev/null +++ b/test/663-odd-dex-size2/info.txt @@ -0,0 +1,15 @@ +Test for two files with an odd size in a vdex file. + +The code in boths file is: + +class Main { + public static void main(String[] args) { + System.out.println("HelloWorld"); + } +} + +The generated dex file was then manually edited to: +1) Add 1 to the size value in the dex header. +2) Add 1 byte to the file. +3) Change the checksum in the dex header. + diff --git a/test/663-odd-dex-size3/663-odd-dex-size3.jar b/test/663-odd-dex-size3/663-odd-dex-size3.jar Binary files differnew file mode 100644 index 0000000000..d23ed5787a --- /dev/null +++ b/test/663-odd-dex-size3/663-odd-dex-size3.jar diff --git a/test/663-odd-dex-size3/build b/test/663-odd-dex-size3/build new file mode 100644 index 0000000000..5636558dad --- /dev/null +++ b/test/663-odd-dex-size3/build @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Nothing to do diff --git a/test/663-odd-dex-size3/expected.txt b/test/663-odd-dex-size3/expected.txt new file mode 100644 index 0000000000..3da1ec26e9 --- /dev/null +++ b/test/663-odd-dex-size3/expected.txt @@ -0,0 +1 @@ +HelloWorld diff --git a/test/663-odd-dex-size3/info.txt b/test/663-odd-dex-size3/info.txt new file mode 100644 index 0000000000..256c77de9c --- /dev/null +++ b/test/663-odd-dex-size3/info.txt @@ -0,0 +1,19 @@ +Test for a dex file with an odd size followed by an aligned dex file. + +The code in classes.dex is: + +class Main { + public static void main(String[] args) { + System.out.println("HelloWorld"); + } +} + +The generated dex file was then manually edited to: +1) Add 1 to the size value in the dex header. +2) Add 1 byte to the file. +3) Change the checksum in the dex header. + +The code in classes2.dex is: + +class Foo { +} diff --git a/test/663-odd-dex-size4/663-odd-dex-size4.jar b/test/663-odd-dex-size4/663-odd-dex-size4.jar Binary files differnew file mode 100644 index 0000000000..d229663605 --- /dev/null +++ b/test/663-odd-dex-size4/663-odd-dex-size4.jar diff --git a/test/663-odd-dex-size4/build b/test/663-odd-dex-size4/build new file mode 100644 index 0000000000..5636558dad --- /dev/null +++ b/test/663-odd-dex-size4/build @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Nothing to do diff --git a/test/663-odd-dex-size4/expected.txt b/test/663-odd-dex-size4/expected.txt new file mode 100644 index 0000000000..3da1ec26e9 --- /dev/null +++ b/test/663-odd-dex-size4/expected.txt @@ -0,0 +1 @@ +HelloWorld diff --git a/test/663-odd-dex-size4/info.txt b/test/663-odd-dex-size4/info.txt new file mode 100644 index 0000000000..2c34557639 --- /dev/null +++ b/test/663-odd-dex-size4/info.txt @@ -0,0 +1,19 @@ +Test for an aligned dex file followed by a dex file with an odd size. + +The code in classes.dex is: + +class Foo { +} + +The code in classes2.dex is: + +class Main { + public static void main(String[] args) { + System.out.println("HelloWorld"); + } +} + +The generated dex file was then manually edited to: +1) Add 1 to the size value in the dex header. +2) Add 1 byte to the file. +3) Change the checksum in the dex header. diff --git a/test/706-checker-scheduler/run b/test/706-checker-scheduler/run new file mode 100644 index 0000000000..5ffc303b2d --- /dev/null +++ b/test/706-checker-scheduler/run @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use secondary switch to add secondary dex file to class path. +exec ${RUN} "${@}" --secondary diff --git a/test/706-checker-scheduler/src-dex2oat-unresolved/UnresolvedClass.java b/test/706-checker-scheduler/src-dex2oat-unresolved/UnresolvedClass.java new file mode 100644 index 0000000000..4faa12a335 --- /dev/null +++ b/test/706-checker-scheduler/src-dex2oat-unresolved/UnresolvedClass.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class UnresolvedClass { + public static int staticInt; + public int instanceInt; +} + diff --git a/test/706-checker-scheduler/src/Main.java b/test/706-checker-scheduler/src/Main.java index a68565b552..08a23a7fbc 100644 --- a/test/706-checker-scheduler/src/Main.java +++ b/test/706-checker-scheduler/src/Main.java @@ -31,6 +31,7 @@ public class Main { public ExampleObj my_obj; public static int number1; public static int number2; + public static volatile int number3; /// CHECK-START-ARM64: int Main.arrayAccess() scheduler (before) /// CHECK: <<Const1:i\d+>> IntConstant 1 @@ -340,6 +341,87 @@ public class Main { } } + /// CHECK-START-ARM: void Main.accessFieldsVolatile() scheduler (before) + /// CHECK-START-ARM64: void Main.accessFieldsVolatile() scheduler (before) + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: StaticFieldGet + /// CHECK: Add + /// CHECK: StaticFieldSet + /// CHECK: StaticFieldGet + /// CHECK: Add + /// CHECK: StaticFieldSet + + /// CHECK-START-ARM: void Main.accessFieldsVolatile() scheduler (after) + /// CHECK-START-ARM64: void Main.accessFieldsVolatile() scheduler (after) + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: StaticFieldGet + /// CHECK: Add + /// CHECK: StaticFieldSet + /// CHECK: StaticFieldGet + /// CHECK: Add + /// CHECK: StaticFieldSet + + public void accessFieldsVolatile() { + my_obj = new ExampleObj(1, 2); + for (int i = 0; i < 10; i++) { + my_obj.n1++; + my_obj.n2++; + number1++; + number3++; + } + } + + /// CHECK-START-ARM: void Main.accessFieldsUnresolved() scheduler (before) + /// CHECK-START-ARM64: void Main.accessFieldsUnresolved() scheduler (before) + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: UnresolvedInstanceFieldGet + /// CHECK: Add + /// CHECK: UnresolvedInstanceFieldSet + /// CHECK: UnresolvedStaticFieldGet + /// CHECK: Add + /// CHECK: UnresolvedStaticFieldSet + + /// CHECK-START-ARM: void Main.accessFieldsUnresolved() scheduler (after) + /// CHECK-START-ARM64: void Main.accessFieldsUnresolved() scheduler (after) + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: InstanceFieldGet + /// CHECK: Add + /// CHECK: InstanceFieldSet + /// CHECK: UnresolvedInstanceFieldGet + /// CHECK: Add + /// CHECK: UnresolvedInstanceFieldSet + /// CHECK: UnresolvedStaticFieldGet + /// CHECK: Add + /// CHECK: UnresolvedStaticFieldSet + + public void accessFieldsUnresolved() { + my_obj = new ExampleObj(1, 2); + UnresolvedClass unresolved_obj = new UnresolvedClass(); + for (int i = 0; i < 10; i++) { + my_obj.n1++; + my_obj.n2++; + unresolved_obj.instanceInt++; + UnresolvedClass.staticInt++; + } + } + /// CHECK-START-ARM64: int Main.intDiv(int) scheduler (before) /// CHECK: Sub /// CHECK: DivZeroCheck diff --git a/test/common/runtime_state.cc b/test/common/runtime_state.cc index 7c0ed691b6..60c7315b6f 100644 --- a/test/common/runtime_state.cc +++ b/test/common/runtime_state.cc @@ -254,4 +254,17 @@ extern "C" JNIEXPORT int JNICALL Java_Main_numberOfDeoptimizations(JNIEnv*, jcla return Runtime::Current()->GetNumberOfDeoptimizations(); } +extern "C" JNIEXPORT void JNICALL Java_Main_fetchProfiles(JNIEnv*, jclass) { + jit::Jit* jit = GetJitIfEnabled(); + if (jit == nullptr) { + return; + } + jit::JitCodeCache* code_cache = jit->GetCodeCache(); + std::vector<ProfileMethodInfo> unused_vector; + std::set<std::string> unused_locations; + unused_locations.insert("fake_location"); + ScopedObjectAccess soa(Thread::Current()); + code_cache->GetProfiledMethods(unused_locations, unused_vector); +} + } // namespace art diff --git a/test/knownfailures.json b/test/knownfailures.json index bd9591a8a8..a8191bbb23 100644 --- a/test/knownfailures.json +++ b/test/knownfailures.json @@ -713,8 +713,16 @@ "--no-dex2oat do not create"] }, { - "tests": "961-default-iface-resolution-gen", + "tests": ["961-default-iface-resolution-gen", + "964-default-iface-init-gen", + "968-default-partial-compile-gen"], "env_vars": {"SANITIZE_HOST": "address"}, "description": ["Test hits dex2oat watchdog timeout (60sec) on art-asan"] + }, + { + "tests": "662-regression-alias", + "variant": "target", + "description": ["disable until ARM scheduling/aliasing bug is fixed."], + "bug": "b/64018485" } ] diff --git a/test/ti-agent/trace_helper.cc b/test/ti-agent/trace_helper.cc index 7a9d1e0d92..1f8ceff91c 100644 --- a/test/ti-agent/trace_helper.cc +++ b/test/ti-agent/trace_helper.cc @@ -388,10 +388,12 @@ extern "C" JNIEXPORT void JNICALL Java_art_Trace_enableTracing( data->single_step = single_step != nullptr ? env->FromReflectedMethod(single_step) : nullptr; data->in_callback = false; - void* old_data = nullptr; - if (JvmtiErrorToException(env, jvmti_env, jvmti_env->GetEnvironmentLocalStorage(&old_data))) { + TraceData* old_data = nullptr; + if (JvmtiErrorToException(env, jvmti_env, + jvmti_env->GetEnvironmentLocalStorage( + reinterpret_cast<void**>(&old_data)))) { return; - } else if (old_data != nullptr) { + } else if (old_data != nullptr && old_data->test_klass != nullptr) { ScopedLocalRef<jclass> rt_exception(env, env->FindClass("java/lang/RuntimeException")); env->ThrowNew(rt_exception.get(), "Environment already has local storage set!"); return; @@ -455,6 +457,21 @@ extern "C" JNIEXPORT void JNICALL Java_art_Trace_enableTracing( extern "C" JNIEXPORT void JNICALL Java_art_Trace_disableTracing( JNIEnv* env, jclass klass ATTRIBUTE_UNUSED, jthread thr) { + TraceData* data = nullptr; + if (JvmtiErrorToException( + env, jvmti_env, jvmti_env->GetEnvironmentLocalStorage(reinterpret_cast<void**>(&data)))) { + return; + } + // If data is null then we haven't ever enabled tracing so we don't need to do anything. + if (data == nullptr || data->test_klass == nullptr) { + return; + } + env->DeleteGlobalRef(data->test_klass); + if (env->ExceptionCheck()) { + return; + } + // Clear test_klass so we know this isn't being used + data->test_klass = nullptr; if (JvmtiErrorToException(env, jvmti_env, jvmti_env->SetEventNotificationMode(JVMTI_DISABLE, JVMTI_EVENT_FIELD_ACCESS, @@ -23,6 +23,7 @@ LAUNCH_WRAPPER= LIBART=libart.so JIT_PROFILE="no" VERBOSE="no" +CLEAN_OAT_FILES="yes" EXTRA_OPTIONS=() # Follow all sym links to get the program name. @@ -87,6 +88,7 @@ Supported OPTIONS include: report upon completion. --profile Run with profiling, then run using profile data. --verbose Run script verbosely. + --no-clean Don't cleanup oat directories. The ART_OPTIONS are passed directly to the Android Runtime. @@ -176,12 +178,14 @@ cleanup_oat_directory() { # Input: Command line arguments to the art script. # e.g. -cp foo/classes.dex:bar/classes.dex would delete (foo/oat bar/oat) directories. cleanup_oat_directory_for_classpath() { - # First try: Use $CLASSPATH environment variable. - parse_classpath "$CLASSPATH" - # Second try: Look for latest -cp or -classpath arg which will take precedence. - find_cp_in_args "$@" + if [ "$CLEAN_OAT_FILES" = "yes" ]; then + # First try: Use $CLASSPATH environment variable. + parse_classpath "$CLASSPATH" + # Second try: Look for latest -cp or -classpath arg which will take precedence. + find_cp_in_args "$@" - cleanup_oat_directory "${PARSE_CLASSPATH_RESULT[@]}" + cleanup_oat_directory "${PARSE_CLASSPATH_RESULT[@]}" + fi } # Attempt to find $ANDROID_ROOT/framework/<isa>/core.art' without knowing what <isa> is. @@ -291,6 +295,9 @@ while [[ "$1" = "-"* ]]; do --verbose) VERBOSE="yes" ;; + --no-clean) + CLEAN_OAT_FILES="no" + ;; --*) echo "unknown option: $1" 1>&2 usage diff --git a/tools/cpp-define-generator/offset_runtime.def b/tools/cpp-define-generator/offset_runtime.def index 41e7e40af5..1d5ce7dd49 100644 --- a/tools/cpp-define-generator/offset_runtime.def +++ b/tools/cpp-define-generator/offset_runtime.def @@ -40,6 +40,10 @@ DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_REFS_ONLY, art::CalleeSaveType::kSaveRefs DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_REFS_AND_ARGS, art::CalleeSaveType::kSaveRefsAndArgs) // Offset of field Runtime::callee_save_methods_[kSaveEverything] DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING, art::CalleeSaveType::kSaveEverything) +// Offset of field Runtime::callee_save_methods_[kSaveEverythingForClinit] +DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING_FOR_CLINIT, art::CalleeSaveType::kSaveEverythingForClinit) +// Offset of field Runtime::callee_save_methods_[kSaveEverythingForSuspendCheck] +DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING_FOR_SUSPEND_CHECK, art::CalleeSaveType::kSaveEverythingForSuspendCheck) #undef DEFINE_RUNTIME_CALLEE_SAVE_OFFSET #include "common_undef.def" // undef DEFINE_OFFSET_EXPR diff --git a/tools/dexfuzz/README b/tools/dexfuzz/README index 1f74262eb4..fff5473327 100644 --- a/tools/dexfuzz/README +++ b/tools/dexfuzz/README @@ -139,6 +139,7 @@ InstructionDuplicator 80 InstructionSwapper 80 InvokeChanger 30 NewArrayLengthChanger 50 +NewInstanceChanger 10 NewMethodCaller 10 NonsenseStringPrinter 10 OppositeBranchChanger 40 diff --git a/tools/dexfuzz/src/dexfuzz/DexFuzz.java b/tools/dexfuzz/src/dexfuzz/DexFuzz.java index 2b3b8e7753..1e37def905 100644 --- a/tools/dexfuzz/src/dexfuzz/DexFuzz.java +++ b/tools/dexfuzz/src/dexfuzz/DexFuzz.java @@ -33,9 +33,9 @@ import dexfuzz.listeners.UpdatingConsoleListener; * Entrypoint class for dexfuzz. */ public class DexFuzz { - // Last version update 1.7: changed the likelihood of RegisterClobber. + // Last version update 1.8: Added a new mutation called NewInstanceChanger. private static int majorVersion = 1; - private static int minorVersion = 7; + private static int minorVersion = 8; private static int seedChangeVersion = 0; /** diff --git a/tools/dexfuzz/src/dexfuzz/program/Program.java b/tools/dexfuzz/src/dexfuzz/program/Program.java index bb2f4c059d..c6fa6c4151 100644 --- a/tools/dexfuzz/src/dexfuzz/program/Program.java +++ b/tools/dexfuzz/src/dexfuzz/program/Program.java @@ -32,6 +32,7 @@ import dexfuzz.program.mutators.InstructionDuplicator; import dexfuzz.program.mutators.InstructionSwapper; import dexfuzz.program.mutators.InvokeChanger; import dexfuzz.program.mutators.NewArrayLengthChanger; +import dexfuzz.program.mutators.NewInstanceChanger; import dexfuzz.program.mutators.NewMethodCaller; import dexfuzz.program.mutators.NonsenseStringPrinter; import dexfuzz.program.mutators.OppositeBranchChanger; @@ -54,6 +55,7 @@ import dexfuzz.rawdex.MethodIdItem; import dexfuzz.rawdex.ProtoIdItem; import dexfuzz.rawdex.RawDexFile; import dexfuzz.rawdex.TypeIdItem; +import dexfuzz.rawdex.TypeList; import dexfuzz.rawdex.formats.ContainsPoolIndex.PoolIndexKind; import java.io.BufferedReader; @@ -204,6 +206,7 @@ public class Program { registerMutator(new InstructionSwapper(rng, mutationStats, mutations)); registerMutator(new InvokeChanger(rng, mutationStats, mutations)); registerMutator(new NewArrayLengthChanger(rng, mutationStats, mutations)); + registerMutator(new NewInstanceChanger(rng, mutationStats, mutations)); registerMutator(new NewMethodCaller(rng, mutationStats, mutations)); registerMutator(new NonsenseStringPrinter(rng, mutationStats, mutations)); registerMutator(new OppositeBranchChanger(rng, mutationStats, mutations)); @@ -609,4 +612,45 @@ public class Program { fieldIdx)); return null; } -} + + /** + * Used to convert the type index into string format. + * @param typeIdx + * @return string format of type index. + */ + public String getTypeString(int typeIdx) { + TypeIdItem typeIdItem = rawDexFile.typeIds.get(typeIdx); + return rawDexFile.stringDatas.get(typeIdItem.descriptorIdx).getString(); + } + + /** + * Used to convert the method index into string format. + * @param methodIdx + * @return string format of method index. + */ + public String getMethodString(int methodIdx) { + MethodIdItem methodIdItem = rawDexFile.methodIds.get(methodIdx); + return rawDexFile.stringDatas.get(methodIdItem.nameIdx).getString(); + } + + /** + * Used to convert methodID to string format of method proto. + * @param methodIdx + * @return string format of shorty. + */ + public String getMethodProto(int methodIdx) { + MethodIdItem methodIdItem = rawDexFile.methodIds.get(methodIdx); + ProtoIdItem protoIdItem = rawDexFile.protoIds.get(methodIdItem.protoIdx); + + if (!protoIdItem.parametersOff.pointsToSomething()) { + return "()" + getTypeString(protoIdItem.returnTypeIdx); + } + + TypeList typeList = (TypeList) protoIdItem.parametersOff.getPointedToItem(); + String typeItem = "("; + for (int i= 0; i < typeList.size; i++) { + typeItem = typeItem + typeList.list[i]; + } + return typeItem + ")" + getTypeString(protoIdItem.returnTypeIdx); + } +}
\ No newline at end of file diff --git a/tools/dexfuzz/src/dexfuzz/program/mutators/NewInstanceChanger.java b/tools/dexfuzz/src/dexfuzz/program/mutators/NewInstanceChanger.java new file mode 100644 index 0000000000..cbf79e34f0 --- /dev/null +++ b/tools/dexfuzz/src/dexfuzz/program/mutators/NewInstanceChanger.java @@ -0,0 +1,218 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dexfuzz.program.mutators; + +import dexfuzz.Log; +import dexfuzz.MutationStats; +import dexfuzz.program.MInsn; +import dexfuzz.program.MutatableCode; +import dexfuzz.program.Mutation; +import dexfuzz.rawdex.Opcode; +import dexfuzz.rawdex.formats.ContainsPoolIndex; +import dexfuzz.rawdex.formats.ContainsPoolIndex.PoolIndexKind; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +/** + * Mutator NewInstanceChanger changes the new instance type in a method to + * any random type from the pool. + */ +public class NewInstanceChanger extends CodeMutator { + + /** + * Every CodeMutator has an AssociatedMutation, representing the + * mutation that this CodeMutator can perform, to allow separate + * generateMutation() and applyMutation() phases, allowing serialization. + */ + public static class AssociatedMutation extends Mutation { + public int newInstanceToChangeIdx; + public int newInstanceTypeIdx; + + @Override + public String getString() { + StringBuilder builder = new StringBuilder(); + builder.append(newInstanceToChangeIdx).append(" "); + builder.append(newInstanceTypeIdx); + return builder.toString(); + } + + @Override + public void parseString(String[] elements) { + newInstanceToChangeIdx = Integer.parseInt(elements[2]); + newInstanceTypeIdx = Integer.parseInt(elements[3]); + } + } + + // The following two methods are here for the benefit of MutationSerializer, + // so it can create a CodeMutator and get the correct associated Mutation, as it + // reads in mutations from a dump of mutations. + @Override + public Mutation getNewMutation() { + return new AssociatedMutation(); + } + + public NewInstanceChanger() {} + + public NewInstanceChanger(Random rng, MutationStats stats, List<Mutation> mutations) { + super(rng, stats, mutations); + likelihood = 10; + } + + // A cache that should only exist between generateMutation() and applyMutation(), + // or be created at the start of applyMutation(), if we're reading in mutations from + // a file. + private List<MInsn> newInstanceCachedInsns = null; + + private void generateCachedNewInstanceInsns(MutatableCode mutatableCode) { + if (newInstanceCachedInsns != null) { + return; + } + + newInstanceCachedInsns = new ArrayList<MInsn>(); + + for (MInsn mInsn : mutatableCode.getInstructions()) { + if (mInsn.insn.info.opcode == Opcode.NEW_INSTANCE) { + newInstanceCachedInsns.add(mInsn); + } + } + } + + @Override + protected boolean canMutate(MutatableCode mutatableCode) { + // Cannot change the pool index with only one type. + if (mutatableCode.program.getTotalPoolIndicesByKind(PoolIndexKind.Type) < 2) { + Log.debug("Cannot mutate, only one type, skipping..."); + return false; + } + + for (MInsn mInsn : mutatableCode.getInstructions()) { + if (mInsn.insn.info.opcode == Opcode.NEW_INSTANCE) { + return true; + } + } + Log.debug("No New Instance in method, skipping..."); + return false; + } + + @Override + protected Mutation generateMutation(MutatableCode mutatableCode) { + generateCachedNewInstanceInsns(mutatableCode); + + int newInstanceIdxInCache = rng.nextInt(newInstanceCachedInsns.size()); + MInsn newInstanceInsn = newInstanceCachedInsns.get(newInstanceIdxInCache); + int oldTypeIdx = (int) newInstanceInsn.insn.vregB; + int newTypeIdx = 0; + int totalPoolIndices = mutatableCode.program.getTotalPoolIndicesByKind(PoolIndexKind.Type); + if (totalPoolIndices < 2) { + Log.errorAndQuit("Less than two types present, quitting..."); + } + + while (newTypeIdx == oldTypeIdx) { + newTypeIdx = rng.nextInt(totalPoolIndices); + } + + AssociatedMutation mutation = new AssociatedMutation(); + mutation.setup(this.getClass(), mutatableCode); + mutation.newInstanceToChangeIdx = newInstanceIdxInCache; + mutation.newInstanceTypeIdx = newTypeIdx; + return mutation; + } + + @Override + protected void applyMutation(Mutation uncastMutation) { + // Cast the Mutation to our AssociatedMutation, so we can access its fields. + AssociatedMutation mutation = (AssociatedMutation) uncastMutation; + MutatableCode mutatableCode = mutation.mutatableCode; + + generateCachedNewInstanceInsns(mutatableCode); + + MInsn newInstanceInsn = newInstanceCachedInsns.get(mutation.newInstanceToChangeIdx); + + ContainsPoolIndex poolIndex = ((ContainsPoolIndex)newInstanceInsn.insn.info.format); + + poolIndex.setPoolIndex(newInstanceInsn.insn, mutation.newInstanceTypeIdx); + + Log.info("Changed the type of " + newInstanceInsn.toString() + + " to " + mutation.newInstanceTypeIdx); + + int foundNewInstanceInsnIdx = + foundInsnIdx(mutatableCode, newInstanceCachedInsns.get(mutation.newInstanceToChangeIdx)); + + changeInvokeDirect(foundNewInstanceInsnIdx, mutation); + + stats.incrementStat("Changed new instance."); + + // Clear cache. + newInstanceCachedInsns = null; + } + + /** + * Try to find the invoke-direct/ invoke-direct-range instruction that follows + * the new instance instruction and change the method ID of the instruction. + * @param foundInsnIdx + * @param uncastMutation + */ + protected void changeInvokeDirect(int foundInsnIdx, Mutation uncastMutation) { + AssociatedMutation mutation = (AssociatedMutation) uncastMutation; + MutatableCode mutatableCode = mutation.mutatableCode; + if (foundInsnIdx == -1 || + foundInsnIdx + 1 == mutatableCode.getInstructionCount()) { + return; + } + + MInsn insn = mutatableCode.getInstructionAt(foundInsnIdx + 1); + if (isInvokeInst(insn)) { + ContainsPoolIndex poolIndex =((ContainsPoolIndex)insn.insn.info.format); + long oldMethodIdx = poolIndex.getPoolIndex(insn.insn); + String className = mutatableCode.program.getTypeString(mutation.newInstanceTypeIdx); + String methodName = mutatableCode.program.getMethodString((int) oldMethodIdx); + String shorty = mutatableCode.program.getMethodProto((int) oldMethodIdx); + + // Matches the type of the invoke with the randomly changed type of the prior new-instance. + // This might create a lot of verification failures but still works many times. + // TODO: Work on generating a program which finds a valid type. + int methodId = mutatableCode.program.getNewItemCreator(). + findOrCreateMethodId(className, methodName, shorty); + + poolIndex.setPoolIndex(insn.insn, mutation.newInstanceTypeIdx); + + insn.insn.vregB = methodId; + + Log.info("Changed " + oldMethodIdx + " to " + methodId); + } + } + + protected boolean isInvokeInst(MInsn mInsn) { + return (mInsn.insn.info.opcode == Opcode.INVOKE_DIRECT || + mInsn.insn.info.opcode == Opcode.INVOKE_DIRECT_RANGE); + } + + // Check if there is an new instance instruction, and if found, return the index. + // If not, return -1. + protected int foundInsnIdx(MutatableCode mutatableCode, MInsn newInstanceInsn) { + int i = 0; + for (MInsn mInsn : mutatableCode.getInstructions()) { + if (mInsn == newInstanceInsn) { + return i; + } + i++; + } + return -1; + } +} diff --git a/tools/libcore_gcstress_debug_failures.txt b/tools/libcore_gcstress_debug_failures.txt index 66bc252055..5806b6107f 100644 --- a/tools/libcore_gcstress_debug_failures.txt +++ b/tools/libcore_gcstress_debug_failures.txt @@ -8,9 +8,11 @@ description: "Timeouts on target with gcstress and debug.", result: EXEC_FAILED, modes: [device], - names: ["libcore.icu.TransliteratorTest#testAll", + names: ["jsr166.CompletableFutureTest#testCompleteOnTimeout_completed", + "libcore.icu.TransliteratorTest#testAll", "libcore.icu.RelativeDateTimeFormatterTest#test_bug25821045", "libcore.java.lang.ref.ReferenceQueueTest#testRemoveWithDelayedResultAndTimeout", + "libcore.java.lang.ref.ReferenceQueueTest#testRemoveWithDelayedResultAndNoTimeout", "libcore.java.util.TimeZoneTest#testSetDefaultDeadlock", "org.apache.harmony.tests.java.util.TimerTest#testThrowingTaskKillsTimerThread"] } diff --git a/tools/run-jdwp-tests.sh b/tools/run-jdwp-tests.sh index d2322bb3a9..355646b713 100755 --- a/tools/run-jdwp-tests.sh +++ b/tools/run-jdwp-tests.sh @@ -122,6 +122,9 @@ while true; do fi done +# Make sure the debuggee doesn't clean up what the debugger has generated. +art_debugee="$art_debugee --no-clean" + # For the host: # # If, on the other hand, there is a variant set, use it to modify the art_debugee parameter to |