Merge changes Ic119441c,I83b96b41
* changes:
optimizing: Add statistics for # of constructor fences added/removed
optimizing: Refactor statistics to use OptimizingCompilerStats helper
diff --git a/build/Android.cpplint.mk b/build/Android.cpplint.mk
index 2688c04..247f4e3 100644
--- a/build/Android.cpplint.mk
+++ b/build/Android.cpplint.mk
@@ -18,7 +18,9 @@
ART_CPPLINT := $(LOCAL_PATH)/tools/cpplint.py
ART_CPPLINT_FILTER := --filter=-whitespace/line_length,-build/include,-readability/function,-readability/streams,-readability/todo,-runtime/references,-runtime/sizeof,-runtime/threadsafe_fn,-runtime/printf
-ART_CPPLINT_FLAGS := --root=$(TOP)
+# Use `pwd` instead of $TOP for root, $TOP is always . and --root doesn't seem
+# to work with a relative path (b/34787652).
+ART_CPPLINT_FLAGS := --root=`pwd`
ART_CPPLINT_QUIET := --quiet
ART_CPPLINT_INGORED := \
runtime/elf.h \
diff --git a/compiler/dex/quick_compiler_callbacks.cc b/compiler/dex/quick_compiler_callbacks.cc
index 872f7ea..c7e9f4f 100644
--- a/compiler/dex/quick_compiler_callbacks.cc
+++ b/compiler/dex/quick_compiler_callbacks.cc
@@ -16,6 +16,7 @@
#include "quick_compiler_callbacks.h"
+#include "driver/compiler_driver.h"
#include "verification_results.h"
#include "verifier/method_verifier-inl.h"
@@ -33,4 +34,17 @@
}
}
+bool QuickCompilerCallbacks::CanAssumeVerified(ClassReference ref) {
+ // If we don't have class unloading enabled in the compiler, we will never see class that were
+ // previously verified. Return false to avoid overhead from the lookup in the compiler driver.
+ if (!does_class_unloading_) {
+ return false;
+ }
+ DCHECK(compiler_driver_ != nullptr);
+ // In the case of the quicken filter: avoiding verification of quickened instructions, which the
+ // verifier doesn't currently support.
+ // In the case of the verify filter, avoiding verifiying twice.
+ return compiler_driver_->CanAssumeVerified(ref);
+}
+
} // namespace art
diff --git a/compiler/dex/quick_compiler_callbacks.h b/compiler/dex/quick_compiler_callbacks.h
index a3a6c09..578aff4 100644
--- a/compiler/dex/quick_compiler_callbacks.h
+++ b/compiler/dex/quick_compiler_callbacks.h
@@ -22,6 +22,7 @@
namespace art {
+class CompilerDriver;
class VerificationResults;
class QuickCompilerCallbacks FINAL : public CompilerCallbacks {
@@ -53,8 +54,19 @@
verification_results_ = verification_results;
}
+ bool CanAssumeVerified(ClassReference ref) OVERRIDE;
+
+ void SetDoesClassUnloading(bool does_class_unloading, CompilerDriver* compiler_driver)
+ OVERRIDE {
+ does_class_unloading_ = does_class_unloading;
+ compiler_driver_ = compiler_driver;
+ DCHECK(!does_class_unloading || compiler_driver_ != nullptr);
+ }
+
private:
VerificationResults* verification_results_ = nullptr;
+ bool does_class_unloading_ = false;
+ CompilerDriver* compiler_driver_ = nullptr;
std::unique_ptr<verifier::VerifierDeps> verifier_deps_;
};
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index bd530ac..037e458 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -2879,9 +2879,9 @@
bool CompilerDriver::GetCompiledClass(ClassReference ref, mirror::Class::Status* status) const {
DCHECK(status != nullptr);
// The table doesn't know if something wasn't inserted. For this case it will return
- // kStatusNotReady. To handle this, just assume anything not verified is not compiled.
+ // kStatusNotReady. To handle this, just assume anything we didn't try to verify is not compiled.
if (!compiled_classes_.Get(DexFileReference(ref.first, ref.second), status) ||
- *status < mirror::Class::kStatusVerified) {
+ *status < mirror::Class::kStatusRetryVerificationAtRuntime) {
return false;
}
return true;
@@ -3040,4 +3040,10 @@
}
}
+bool CompilerDriver::CanAssumeVerified(ClassReference ref) const {
+ mirror::Class::Status existing = mirror::Class::kStatusNotReady;
+ compiled_classes_.Get(DexFileReference(ref.first, ref.second), &existing);
+ return existing >= mirror::Class::kStatusVerified;
+}
+
} // namespace art
diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h
index d9886a2..11808c1 100644
--- a/compiler/driver/compiler_driver.h
+++ b/compiler/driver/compiler_driver.h
@@ -22,6 +22,8 @@
#include <unordered_set>
#include <vector>
+#include "android-base/strings.h"
+
#include "arch/instruction_set.h"
#include "base/array_ref.h"
#include "base/bit_utils.h"
@@ -377,6 +379,16 @@
return profile_compilation_info_;
}
+ bool CanAssumeVerified(ClassReference ref) const;
+
+ // Is `boot_image_filename` the name of a core image (small boot
+ // image used for ART testing only)?
+ static bool IsCoreImageFilename(const std::string& boot_image_filename) {
+ // TODO: This is under-approximating...
+ return android::base::EndsWith(boot_image_filename, "core.art")
+ || android::base::EndsWith(boot_image_filename, "core-optimizing.art");
+ }
+
private:
void PreCompile(jobject class_loader,
const std::vector<const DexFile*>& dex_files,
diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc
index fee6afb..392d57c 100644
--- a/compiler/driver/compiler_driver_test.cc
+++ b/compiler/driver/compiler_driver_test.cc
@@ -23,6 +23,7 @@
#include "art_method-inl.h"
#include "class_linker-inl.h"
#include "common_compiler_test.h"
+#include "compiler_callbacks.h"
#include "dex_file.h"
#include "dex_file_types.h"
#include "gc/heap.h"
@@ -366,6 +367,49 @@
CheckVerifiedClass(class_loader, "LSecond;");
}
+// Test that a class of status kStatusRetryVerificationAtRuntime is indeed recorded that way in the
+// driver.
+// Test that checks that classes can be assumed as verified if unloading mode is enabled and
+// the class status is at least verified.
+TEST_F(CompilerDriverVerifyTest, RetryVerifcationStatusCheckVerified) {
+ Thread* const self = Thread::Current();
+ jobject class_loader;
+ std::vector<const DexFile*> dex_files;
+ const DexFile* dex_file = nullptr;
+ {
+ ScopedObjectAccess soa(self);
+ class_loader = LoadDex("ProfileTestMultiDex");
+ ASSERT_NE(class_loader, nullptr);
+ dex_files = GetDexFiles(class_loader);
+ ASSERT_GT(dex_files.size(), 0u);
+ dex_file = dex_files.front();
+ }
+ compiler_driver_->SetDexFilesForOatFile(dex_files);
+ callbacks_->SetDoesClassUnloading(true, compiler_driver_.get());
+ ClassReference ref(dex_file, 0u);
+ // Test that the status is read from the compiler driver as expected.
+ for (size_t i = mirror::Class::kStatusRetryVerificationAtRuntime;
+ i < mirror::Class::kStatusMax;
+ ++i) {
+ const mirror::Class::Status expected_status = static_cast<mirror::Class::Status>(i);
+ // Skip unsupported status that are not supposed to be ever recorded.
+ if (expected_status == mirror::Class::kStatusVerifyingAtRuntime ||
+ expected_status == mirror::Class::kStatusInitializing) {
+ continue;
+ }
+ compiler_driver_->RecordClassStatus(ref, expected_status);
+ mirror::Class::Status status = {};
+ ASSERT_TRUE(compiler_driver_->GetCompiledClass(ref, &status));
+ EXPECT_EQ(status, expected_status);
+
+ // Check that we can assume verified if we are a status that is at least verified.
+ if (status >= mirror::Class::kStatusVerified) {
+ // Check that the class can be assumed as verified in the compiler driver.
+ EXPECT_TRUE(callbacks_->CanAssumeVerified(ref)) << status;
+ }
+ }
+}
+
// TODO: need check-cast test (when stub complete & we can throw/catch
} // namespace art
diff --git a/compiler/driver/compiler_options.cc b/compiler/driver/compiler_options.cc
index 3cacc2c..538845d 100644
--- a/compiler/driver/compiler_options.cc
+++ b/compiler/driver/compiler_options.cc
@@ -18,6 +18,8 @@
#include <fstream>
+#include "runtime.h"
+
namespace art {
CompilerOptions::CompilerOptions()
@@ -30,6 +32,7 @@
inline_max_code_units_(kUnsetInlineMaxCodeUnits),
no_inline_from_(nullptr),
boot_image_(false),
+ core_image_(false),
app_image_(false),
top_k_profile_threshold_(kDefaultTopKProfileThreshold),
debuggable_(false),
@@ -55,6 +58,19 @@
// because we don't want to include the PassManagerOptions definition from the header file.
}
+bool CompilerOptions::EmitRunTimeChecksInDebugMode() const {
+ // Run-time checks (e.g. Marking Register checks) are only emitted
+ // in debug mode, and
+ // - when running on device; or
+ // - when running on host, but only
+ // - when compiling the core image (which is used only for testing); or
+ // - when JIT compiling (only relevant for non-native methods).
+ // This is to prevent these checks from being emitted into pre-opted
+ // boot image or apps, as these are compiled with dex2oatd.
+ return kIsDebugBuild &&
+ (kIsTargetBuild || IsCoreImage() || Runtime::Current()->UseJitCompilation());
+}
+
void CompilerOptions::ParseHugeMethodMax(const StringPiece& option, UsageFn Usage) {
ParseUintOption(option, "--huge-method-max", &huge_method_threshold_, Usage);
}
diff --git a/compiler/driver/compiler_options.h b/compiler/driver/compiler_options.h
index b99263d..1e05c4e 100644
--- a/compiler/driver/compiler_options.h
+++ b/compiler/driver/compiler_options.h
@@ -161,6 +161,9 @@
return generate_mini_debug_info_;
}
+ // Should run-time checks be emitted in debug mode?
+ bool EmitRunTimeChecksInDebugMode() const;
+
bool GetGenerateBuildId() const {
return generate_build_id_;
}
@@ -177,10 +180,19 @@
return implicit_suspend_checks_;
}
+ // Are we compiling a boot image?
bool IsBootImage() const {
return boot_image_;
}
+ // Are we compiling a core image (small boot image only used for ART testing)?
+ bool IsCoreImage() const {
+ // Ensure that `core_image_` => `boot_image_`.
+ DCHECK(!core_image_ || boot_image_);
+ return core_image_;
+ }
+
+ // Are we compiling an app image?
bool IsAppImage() const {
return app_image_;
}
@@ -266,6 +278,7 @@
const std::vector<const DexFile*>* no_inline_from_;
bool boot_image_;
+ bool core_image_;
bool app_image_;
// When using a profile file only the top K% of the profiled samples will be compiled.
double top_k_profile_threshold_;
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 9e4971c..115e722 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -1686,6 +1686,10 @@
runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs);
image_methods_[ImageHeader::kSaveEverythingMethod] =
runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything);
+ image_methods_[ImageHeader::kSaveEverythingMethodForClinit] =
+ runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit);
+ image_methods_[ImageHeader::kSaveEverythingMethodForSuspendCheck] =
+ runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck);
// Visit image methods first to have the main runtime methods in the first image.
for (auto* m : image_methods_) {
CHECK(m != nullptr);
diff --git a/compiler/jni/quick/jni_compiler.cc b/compiler/jni/quick/jni_compiler.cc
index b65b93f..e7e4647 100644
--- a/compiler/jni/quick/jni_compiler.cc
+++ b/compiler/jni/quick/jni_compiler.cc
@@ -219,7 +219,9 @@
// Assembler that holds generated instructions
std::unique_ptr<JNIMacroAssembler<kPointerSize>> jni_asm =
GetMacroAssembler<kPointerSize>(&arena, instruction_set, instruction_set_features);
- jni_asm->cfi().SetEnabled(driver->GetCompilerOptions().GenerateAnyDebugInfo());
+ const CompilerOptions& compiler_options = driver->GetCompilerOptions();
+ jni_asm->cfi().SetEnabled(compiler_options.GenerateAnyDebugInfo());
+ jni_asm->SetEmitRunTimeChecksInDebugMode(compiler_options.EmitRunTimeChecksInDebugMode());
// Offsets into data structures
// TODO: if cross compiling these offsets are for the host not the target
diff --git a/compiler/optimizing/bounds_check_elimination.cc b/compiler/optimizing/bounds_check_elimination.cc
index c166deb..2f96cfa 100644
--- a/compiler/optimizing/bounds_check_elimination.cc
+++ b/compiler/optimizing/bounds_check_elimination.cc
@@ -1121,6 +1121,66 @@
}
}
+ void VisitRem(HRem* instruction) OVERRIDE {
+ HInstruction* left = instruction->GetLeft();
+ HInstruction* right = instruction->GetRight();
+
+ // Handle 'i % CONST' format expression in array index, e.g:
+ // array[i % 20];
+ if (right->IsIntConstant()) {
+ int32_t right_const = std::abs(right->AsIntConstant()->GetValue());
+ if (right_const == 0) {
+ return;
+ }
+ // The sign of divisor CONST doesn't affect the sign final value range.
+ // For example:
+ // if (i > 0) {
+ // array[i % 10]; // index value range [0, 9]
+ // array[i % -10]; // index value range [0, 9]
+ // }
+ ValueRange* right_range = new (GetGraph()->GetArena()) ValueRange(
+ GetGraph()->GetArena(),
+ ValueBound(nullptr, 1 - right_const),
+ ValueBound(nullptr, right_const - 1));
+
+ ValueRange* left_range = LookupValueRange(left, left->GetBlock());
+ if (left_range != nullptr) {
+ right_range = left_range->Narrow(right_range);
+ }
+ AssignRange(instruction->GetBlock(), instruction, right_range);
+ return;
+ }
+
+ // Handle following pattern:
+ // i0 NullCheck
+ // i1 ArrayLength[i0]
+ // i2 DivByZeroCheck [i1] <-- right
+ // i3 Rem [i5, i2] <-- we are here.
+ // i4 BoundsCheck [i3,i1]
+ if (right->IsDivZeroCheck()) {
+ // if array_length can pass div-by-zero check,
+ // array_length must be > 0.
+ right = right->AsDivZeroCheck()->InputAt(0);
+ }
+
+ // Handle 'i % array.length' format expression in array index, e.g:
+ // array[(i+7) % array.length];
+ if (right->IsArrayLength()) {
+ ValueBound lower = ValueBound::Min(); // ideally, lower should be '1-array_length'.
+ ValueBound upper = ValueBound(right, -1); // array_length - 1
+ ValueRange* right_range = new (GetGraph()->GetArena()) ValueRange(
+ GetGraph()->GetArena(),
+ lower,
+ upper);
+ ValueRange* left_range = LookupValueRange(left, left->GetBlock());
+ if (left_range != nullptr) {
+ right_range = left_range->Narrow(right_range);
+ }
+ AssignRange(instruction->GetBlock(), instruction, right_range);
+ return;
+ }
+ }
+
void VisitNewArray(HNewArray* new_array) OVERRIDE {
HInstruction* len = new_array->GetLength();
if (!len->IsIntConstant()) {
diff --git a/compiler/optimizing/bounds_check_elimination_test.cc b/compiler/optimizing/bounds_check_elimination_test.cc
index 575e2fc..2aaf058 100644
--- a/compiler/optimizing/bounds_check_elimination_test.cc
+++ b/compiler/optimizing/bounds_check_elimination_test.cc
@@ -951,4 +951,152 @@
ASSERT_TRUE(IsRemoved(bounds_check6));
}
+// int[] array = new int[10];
+// for (int i=0; i<200; i++) {
+// array[i%10] = 10; // Can eliminate
+// array[i%1] = 10; // Can eliminate
+// array[i%200] = 10; // Cannot eliminate
+// array[i%-10] = 10; // Can eliminate
+// array[i%array.length] = 10; // Can eliminate
+// array[param_i%10] = 10; // Can't eliminate, when param_i < 0
+// }
+TEST_F(BoundsCheckEliminationTest, ModArrayBoundsElimination) {
+ HBasicBlock* entry = new (&allocator_) HBasicBlock(graph_);
+ graph_->AddBlock(entry);
+ graph_->SetEntryBlock(entry);
+ HInstruction* param_i = new (&allocator_)
+ HParameterValue(graph_->GetDexFile(), dex::TypeIndex(0), 0, Primitive::kPrimInt);
+ entry->AddInstruction(param_i);
+
+ HInstruction* constant_0 = graph_->GetIntConstant(0);
+ HInstruction* constant_1 = graph_->GetIntConstant(1);
+ HInstruction* constant_10 = graph_->GetIntConstant(10);
+ HInstruction* constant_200 = graph_->GetIntConstant(200);
+ HInstruction* constant_minus_10 = graph_->GetIntConstant(-10);
+
+ HBasicBlock* block = new (&allocator_) HBasicBlock(graph_);
+ graph_->AddBlock(block);
+ entry->AddSuccessor(block);
+ // We pass a bogus constant for the class to avoid mocking one.
+ HInstruction* new_array = new (&allocator_) HNewArray(constant_10, constant_10, 0);
+ block->AddInstruction(new_array);
+ block->AddInstruction(new (&allocator_) HGoto());
+
+ HBasicBlock* loop_header = new (&allocator_) HBasicBlock(graph_);
+ HBasicBlock* loop_body = new (&allocator_) HBasicBlock(graph_);
+ HBasicBlock* exit = new (&allocator_) HBasicBlock(graph_);
+
+ graph_->AddBlock(loop_header);
+ graph_->AddBlock(loop_body);
+ graph_->AddBlock(exit);
+ block->AddSuccessor(loop_header);
+ loop_header->AddSuccessor(exit); // true successor
+ loop_header->AddSuccessor(loop_body); // false successor
+ loop_body->AddSuccessor(loop_header);
+
+ HPhi* phi = new (&allocator_) HPhi(&allocator_, 0, 0, Primitive::kPrimInt);
+ HInstruction* cmp = new (&allocator_) HGreaterThanOrEqual(phi, constant_200);
+ HInstruction* if_inst = new (&allocator_) HIf(cmp);
+ loop_header->AddPhi(phi);
+ loop_header->AddInstruction(cmp);
+ loop_header->AddInstruction(if_inst);
+ phi->AddInput(constant_0);
+
+ //////////////////////////////////////////////////////////////////////////////////
+ // LOOP BODY:
+ // array[i % 10] = 10;
+ HRem* i_mod_10 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_10, 0);
+ HBoundsCheck* bounds_check_i_mod_10 = new (&allocator_) HBoundsCheck(i_mod_10, constant_10, 0);
+ HInstruction* array_set = new (&allocator_) HArraySet(
+ new_array, bounds_check_i_mod_10, constant_10, Primitive::kPrimInt, 0);
+ loop_body->AddInstruction(i_mod_10);
+ loop_body->AddInstruction(bounds_check_i_mod_10);
+ loop_body->AddInstruction(array_set);
+
+ // array[i % 1] = 10;
+ HRem* i_mod_1 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_1, 0);
+ HBoundsCheck* bounds_check_i_mod_1 = new (&allocator_) HBoundsCheck(i_mod_1, constant_10, 0);
+ array_set = new (&allocator_) HArraySet(
+ new_array, bounds_check_i_mod_1, constant_10, Primitive::kPrimInt, 0);
+ loop_body->AddInstruction(i_mod_1);
+ loop_body->AddInstruction(bounds_check_i_mod_1);
+ loop_body->AddInstruction(array_set);
+
+ // array[i % 200] = 10;
+ HRem* i_mod_200 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_1, 0);
+ HBoundsCheck* bounds_check_i_mod_200 = new (&allocator_) HBoundsCheck(i_mod_200, constant_10, 0);
+ array_set = new (&allocator_) HArraySet(
+ new_array, bounds_check_i_mod_200, constant_10, Primitive::kPrimInt, 0);
+ loop_body->AddInstruction(i_mod_200);
+ loop_body->AddInstruction(bounds_check_i_mod_200);
+ loop_body->AddInstruction(array_set);
+
+ // array[i % -10] = 10;
+ HRem* i_mod_minus_10 = new (&allocator_) HRem(Primitive::kPrimInt, phi, constant_minus_10, 0);
+ HBoundsCheck* bounds_check_i_mod_minus_10 = new (&allocator_) HBoundsCheck(
+ i_mod_minus_10, constant_10, 0);
+ array_set = new (&allocator_) HArraySet(
+ new_array, bounds_check_i_mod_minus_10, constant_10, Primitive::kPrimInt, 0);
+ loop_body->AddInstruction(i_mod_minus_10);
+ loop_body->AddInstruction(bounds_check_i_mod_minus_10);
+ loop_body->AddInstruction(array_set);
+
+ // array[i%array.length] = 10;
+ HNullCheck* null_check = new (&allocator_) HNullCheck(new_array, 0);
+ HArrayLength* array_length = new (&allocator_) HArrayLength(null_check, 0);
+ HRem* i_mod_array_length = new (&allocator_) HRem(Primitive::kPrimInt, phi, array_length, 0);
+ HBoundsCheck* bounds_check_i_mod_array_len = new (&allocator_) HBoundsCheck(
+ i_mod_array_length, array_length, 0);
+ array_set = new (&allocator_) HArraySet(
+ null_check, bounds_check_i_mod_array_len, constant_10, Primitive::kPrimInt, 0);
+ loop_body->AddInstruction(null_check);
+ loop_body->AddInstruction(array_length);
+ loop_body->AddInstruction(i_mod_array_length);
+ loop_body->AddInstruction(bounds_check_i_mod_array_len);
+ loop_body->AddInstruction(array_set);
+
+ // array[param_i % 10] = 10;
+ HRem* param_i_mod_10 = new (&allocator_) HRem(Primitive::kPrimInt, param_i, constant_10, 0);
+ HBoundsCheck* bounds_check_param_i_mod_10 = new (&allocator_) HBoundsCheck(
+ param_i_mod_10, constant_10, 0);
+ array_set = new (&allocator_) HArraySet(
+ new_array, bounds_check_param_i_mod_10, constant_10, Primitive::kPrimInt, 0);
+ loop_body->AddInstruction(param_i_mod_10);
+ loop_body->AddInstruction(bounds_check_param_i_mod_10);
+ loop_body->AddInstruction(array_set);
+
+ // array[param_i%array.length] = 10;
+ null_check = new (&allocator_) HNullCheck(new_array, 0);
+ array_length = new (&allocator_) HArrayLength(null_check, 0);
+ HRem* param_i_mod_array_length = new (&allocator_) HRem(
+ Primitive::kPrimInt, param_i, array_length, 0);
+ HBoundsCheck* bounds_check_param_i_mod_array_len = new (&allocator_) HBoundsCheck(
+ param_i_mod_array_length, array_length, 0);
+ array_set = new (&allocator_) HArraySet(
+ null_check, bounds_check_param_i_mod_array_len, constant_10, Primitive::kPrimInt, 0);
+ loop_body->AddInstruction(null_check);
+ loop_body->AddInstruction(array_length);
+ loop_body->AddInstruction(param_i_mod_array_length);
+ loop_body->AddInstruction(bounds_check_param_i_mod_array_len);
+ loop_body->AddInstruction(array_set);
+
+ // i++;
+ HInstruction* add = new (&allocator_) HAdd(Primitive::kPrimInt, phi, constant_1);
+ loop_body->AddInstruction(add);
+ loop_body->AddInstruction(new (&allocator_) HGoto());
+ phi->AddInput(add);
+ //////////////////////////////////////////////////////////////////////////////////
+
+ exit->AddInstruction(new (&allocator_) HExit());
+
+ RunBCE();
+
+ ASSERT_TRUE(IsRemoved(bounds_check_i_mod_10));
+ ASSERT_TRUE(IsRemoved(bounds_check_i_mod_1));
+ ASSERT_TRUE(IsRemoved(bounds_check_i_mod_200));
+ ASSERT_TRUE(IsRemoved(bounds_check_i_mod_minus_10));
+ ASSERT_TRUE(IsRemoved(bounds_check_i_mod_array_len));
+ ASSERT_FALSE(IsRemoved(bounds_check_param_i_mod_10));
+}
+
} // namespace art
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 4999950..3be774a 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -1595,6 +1595,8 @@
__ Str(wzr, MemOperand(sp, GetStackOffsetOfShouldDeoptimizeFlag()));
}
}
+
+ MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void CodeGeneratorARM64::GenerateFrameExit() {
@@ -3587,6 +3589,7 @@
}
if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
if (!codegen_->GoesToNextBlock(block, successor)) {
__ B(codegen_->GetLabelOf(successor));
@@ -4391,6 +4394,7 @@
void InstructionCodeGeneratorARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) {
@@ -4459,6 +4463,8 @@
DCHECK(!codegen_->IsLeafMethod());
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
}
+
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
@@ -4626,6 +4632,7 @@
void InstructionCodeGeneratorARM64::VisitInvokePolymorphic(HInvokePolymorphic* invoke) {
codegen_->GenerateInvokePolymorphicCall(invoke);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeMethodPatch(
@@ -4801,27 +4808,37 @@
DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
if (TryGenerateIntrinsicCode(invoke, codegen_)) {
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
return;
}
- // Ensure that between the BLR (emitted by GenerateStaticOrDirectCall) and RecordPcInfo there
- // are no pools emitted.
- EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
- LocationSummary* locations = invoke->GetLocations();
- codegen_->GenerateStaticOrDirectCall(
- invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
+ {
+ // Ensure that between the BLR (emitted by GenerateStaticOrDirectCall) and RecordPcInfo there
+ // are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
+ LocationSummary* locations = invoke->GetLocations();
+ codegen_->GenerateStaticOrDirectCall(
+ invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
+ }
+
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
if (TryGenerateIntrinsicCode(invoke, codegen_)) {
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
return;
}
- // Ensure that between the BLR (emitted by GenerateVirtualCall) and RecordPcInfo there
- // are no pools emitted.
- EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
- codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
- DCHECK(!codegen_->IsLeafMethod());
+ {
+ // Ensure that between the BLR (emitted by GenerateVirtualCall) and RecordPcInfo there
+ // are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
+ codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
+ DCHECK(!codegen_->IsLeafMethod());
+ }
+
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind(
@@ -4895,6 +4912,7 @@
HLoadClass::LoadKind load_kind = cls->GetLoadKind();
if (load_kind == HLoadClass::LoadKind::kRuntimeCall) {
codegen_->GenerateLoadClassRuntimeCall(cls);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
return;
}
DCHECK(!cls->NeedsAccessCheck());
@@ -4995,6 +5013,7 @@
} else {
__ Bind(slow_path->GetExitLabel());
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
}
@@ -5113,6 +5132,7 @@
codegen_->AddSlowPath(slow_path);
__ Cbz(out.X(), slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
return;
}
case HLoadString::LoadKind::kJitTableAddress: {
@@ -5137,6 +5157,7 @@
__ Mov(calling_convention.GetRegisterAt(0).W(), load->GetStringIndex().index_);
codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc());
CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) {
@@ -5164,6 +5185,7 @@
} else {
CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void LocationsBuilderARM64::VisitMul(HMul* mul) {
@@ -5260,6 +5282,7 @@
CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass());
codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc());
CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>();
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
@@ -5296,6 +5319,7 @@
codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc());
CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void LocationsBuilderARM64::VisitNot(HNot* instruction) {
@@ -5644,6 +5668,7 @@
return;
}
GenerateSuspendCheck(instruction, nullptr);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void LocationsBuilderARM64::VisitThrow(HThrow* instruction) {
@@ -6021,6 +6046,7 @@
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -6074,22 +6100,25 @@
obj.GetCode());
vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data);
- EmissionCheckScope guard(GetVIXLAssembler(),
- (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize);
- vixl::aarch64::Label return_address;
- __ adr(lr, &return_address);
- __ Bind(cbnz_label);
- __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time.
- static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
- "Field LDR must be 1 instruction (4B) before the return address label; "
- " 2 instructions (8B) for heap poisoning.");
- Register ref_reg = RegisterFrom(ref, Primitive::kPrimNot);
- __ ldr(ref_reg, MemOperand(base.X(), offset));
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
+ {
+ EmissionCheckScope guard(GetVIXLAssembler(),
+ (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize);
+ vixl::aarch64::Label return_address;
+ __ adr(lr, &return_address);
+ __ Bind(cbnz_label);
+ __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time.
+ static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
+ "Field LDR must be 1 instruction (4B) before the return address label; "
+ " 2 instructions (8B) for heap poisoning.");
+ Register ref_reg = RegisterFrom(ref, Primitive::kPrimNot);
+ __ ldr(ref_reg, MemOperand(base.X(), offset));
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
+ __ Bind(&return_address);
}
- GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
- __ Bind(&return_address);
+ MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__, /* temp_loc */ LocationFrom(ip1));
return;
}
@@ -6158,19 +6187,22 @@
vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data);
__ Add(temp.X(), obj.X(), Operand(data_offset));
- EmissionCheckScope guard(GetVIXLAssembler(),
- (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize);
- vixl::aarch64::Label return_address;
- __ adr(lr, &return_address);
- __ Bind(cbnz_label);
- __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time.
- static_assert(BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
- "Array LDR must be 1 instruction (4B) before the return address label; "
- " 2 instructions (8B) for heap poisoning.");
- __ ldr(ref_reg, MemOperand(temp.X(), index_reg.X(), LSL, scale_factor));
- DCHECK(!needs_null_check); // The thunk cannot handle the null check.
- GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
- __ Bind(&return_address);
+ {
+ EmissionCheckScope guard(GetVIXLAssembler(),
+ (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize);
+ vixl::aarch64::Label return_address;
+ __ adr(lr, &return_address);
+ __ Bind(cbnz_label);
+ __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time.
+ static_assert(BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
+ "Array LDR must be 1 instruction (4B) before the return address label; "
+ " 2 instructions (8B) for heap poisoning.");
+ __ ldr(ref_reg, MemOperand(temp.X(), index_reg.X(), LSL, scale_factor));
+ DCHECK(!needs_null_check); // The thunk cannot handle the null check.
+ GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
+ __ Bind(&return_address);
+ }
+ MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__, /* temp_loc */ LocationFrom(ip1));
return;
}
@@ -6247,6 +6279,7 @@
GenerateRawReferenceLoad(
instruction, ref, obj, offset, index, scale_factor, needs_null_check, use_load_acquire);
__ Bind(slow_path->GetExitLabel());
+ MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction,
@@ -6303,6 +6336,7 @@
// Fast path: the GC is not marking: nothing to do (the field is
// up-to-date, and we don't need to load the reference).
__ Bind(slow_path->GetExitLabel());
+ MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void CodeGeneratorARM64::GenerateRawReferenceLoad(HInstruction* instruction,
@@ -6381,6 +6415,19 @@
GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
}
+void CodeGeneratorARM64::MaybeGenerateMarkingRegisterCheck(int code, Location temp_loc) {
+ // The following condition is a compile-time one, so it does not have a run-time cost.
+ if (kEmitCompilerReadBarrier && kUseBakerReadBarrier && kIsDebugBuild) {
+ // The following condition is a run-time one; it is executed after the
+ // previous compile-time test, to avoid penalizing non-debug builds.
+ if (GetCompilerOptions().EmitRunTimeChecksInDebugMode()) {
+ UseScratchRegisterScope temps(GetVIXLAssembler());
+ Register temp = temp_loc.IsValid() ? WRegisterFrom(temp_loc) : temps.AcquireW();
+ GetAssembler()->GenerateMarkingRegisterCheck(temp, code);
+ }
+ }
+}
+
void CodeGeneratorARM64::GenerateReadBarrierSlow(HInstruction* instruction,
Location out,
Location ref,
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index 584eead..c339209 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -687,6 +687,22 @@
bool needs_null_check,
bool use_load_acquire);
+ // Emit code checking the status of the Marking Register, and
+ // aborting the program if MR does not match the value stored in the
+ // art::Thread object. Code is only emitted in debug mode and if
+ // CompilerOptions::EmitRunTimeChecksInDebugMode returns true.
+ //
+ // Argument `code` is used to identify the different occurrences of
+ // MaybeGenerateMarkingRegisterCheck in the code generator, and is
+ // passed to the BRK instruction.
+ //
+ // If `temp_loc` is a valid location, it is expected to be a
+ // register and will be used as a temporary to generate code;
+ // otherwise, a temporary will be fetched from the core register
+ // scratch pool.
+ virtual void MaybeGenerateMarkingRegisterCheck(int code,
+ Location temp_loc = Location::NoLocation());
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index 430cdde..3d45dd3 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -94,6 +94,9 @@
// The reserved entrypoint register for link-time generated thunks.
const vixl32::Register kBakerCcEntrypointRegister = r4;
+// Using a base helps identify when we hit Marking Register check breakpoints.
+constexpr int kMarkingRegisterCheckBreakCodeBaseCode = 0x10;
+
#ifdef __
#error "ARM Codegen VIXL macro-assembler macro already defined."
#endif
@@ -2690,6 +2693,8 @@
__ Mov(temp, 0);
GetAssembler()->StoreToOffset(kStoreWord, temp, sp, GetStackOffsetOfShouldDeoptimizeFlag());
}
+
+ MaybeGenerateMarkingRegisterCheck(/* code */ 1);
}
void CodeGeneratorARMVIXL::GenerateFrameExit() {
@@ -2938,6 +2943,7 @@
}
if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 2);
}
if (!codegen_->GoesToNextBlock(block, successor)) {
__ B(codegen_->GetLabelOf(successor));
@@ -3655,6 +3661,7 @@
void InstructionCodeGeneratorARMVIXL::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 3);
}
void LocationsBuilderARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
@@ -3685,12 +3692,15 @@
DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
if (TryGenerateIntrinsicCode(invoke, codegen_)) {
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 4);
return;
}
LocationSummary* locations = invoke->GetLocations();
codegen_->GenerateStaticOrDirectCall(
invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
+
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 5);
}
void LocationsBuilderARMVIXL::HandleInvoke(HInvoke* invoke) {
@@ -3709,11 +3719,14 @@
void InstructionCodeGeneratorARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) {
if (TryGenerateIntrinsicCode(invoke, codegen_)) {
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 6);
return;
}
codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
DCHECK(!codegen_->IsLeafMethod());
+
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 7);
}
void LocationsBuilderARMVIXL::VisitInvokeInterface(HInvokeInterface* invoke) {
@@ -3790,6 +3803,8 @@
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
DCHECK(!codegen_->IsLeafMethod());
}
+
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 8);
}
void LocationsBuilderARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) {
@@ -3798,6 +3813,7 @@
void InstructionCodeGeneratorARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) {
codegen_->GenerateInvokePolymorphicCall(invoke);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 9);
}
void LocationsBuilderARMVIXL::VisitNeg(HNeg* neg) {
@@ -5329,6 +5345,7 @@
codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc());
CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 10);
}
void LocationsBuilderARMVIXL::VisitNewArray(HNewArray* instruction) {
@@ -5348,6 +5365,7 @@
codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc());
CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>();
DCHECK(!codegen_->IsLeafMethod());
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 11);
}
void LocationsBuilderARMVIXL::VisitParameterValue(HParameterValue* instruction) {
@@ -6965,6 +6983,7 @@
return;
}
GenerateSuspendCheck(instruction, nullptr);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 12);
}
void InstructionCodeGeneratorARMVIXL::GenerateSuspendCheck(HSuspendCheck* instruction,
@@ -7326,6 +7345,7 @@
HLoadClass::LoadKind load_kind = cls->GetLoadKind();
if (load_kind == HLoadClass::LoadKind::kRuntimeCall) {
codegen_->GenerateLoadClassRuntimeCall(cls);
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 13);
return;
}
DCHECK(!cls->NeedsAccessCheck());
@@ -7405,6 +7425,7 @@
} else {
__ Bind(slow_path->GetExitLabel());
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 14);
}
}
@@ -7528,6 +7549,7 @@
codegen_->AddSlowPath(slow_path);
__ CompareAndBranchIfZero(out, slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 15);
return;
}
case HLoadString::LoadKind::kJitTableAddress: {
@@ -7548,6 +7570,7 @@
__ Mov(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_);
codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc());
CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 16);
}
static int32_t GetExceptionTlsOffset() {
@@ -8146,6 +8169,7 @@
} else {
CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 17);
}
void LocationsBuilderARMVIXL::VisitAnd(HAnd* instruction) {
@@ -8647,6 +8671,7 @@
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
+ codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 18);
}
void CodeGeneratorARMVIXL::MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations) {
@@ -8711,31 +8736,34 @@
base.GetCode(), obj.GetCode(), narrow);
vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data);
- vixl::EmissionCheckScope guard(
- GetVIXLAssembler(),
- (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes);
- vixl32::Label return_address;
- EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address);
- __ cmp(mr, Operand(0));
- EmitPlaceholderBne(this, bne_label);
- ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset();
- __ ldr(EncodingSize(narrow ? Narrow : Wide), ref_reg, MemOperand(base, offset));
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- // Note: We need a specific width for the unpoisoning NEG.
- if (kPoisonHeapReferences) {
- if (narrow) {
- // The only 16-bit encoding is T1 which sets flags outside IT block (i.e. RSBS, not RSB).
- __ rsbs(EncodingSize(Narrow), ref_reg, ref_reg, Operand(0));
- } else {
- __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0));
+ {
+ vixl::EmissionCheckScope guard(
+ GetVIXLAssembler(),
+ (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes);
+ vixl32::Label return_address;
+ EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address);
+ __ cmp(mr, Operand(0));
+ EmitPlaceholderBne(this, bne_label);
+ ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset();
+ __ ldr(EncodingSize(narrow ? Narrow : Wide), ref_reg, MemOperand(base, offset));
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
}
+ // Note: We need a specific width for the unpoisoning NEG.
+ if (kPoisonHeapReferences) {
+ if (narrow) {
+ // The only 16-bit encoding is T1 which sets flags outside IT block (i.e. RSBS, not RSB).
+ __ rsbs(EncodingSize(Narrow), ref_reg, ref_reg, Operand(0));
+ } else {
+ __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0));
+ }
+ }
+ __ Bind(&return_address);
+ DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(),
+ narrow ? BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET
+ : BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET);
}
- __ Bind(&return_address);
- DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(),
- narrow ? BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET
- : BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET);
+ MaybeGenerateMarkingRegisterCheck(/* code */ 19, /* temp_loc */ LocationFrom(ip));
return;
}
@@ -8796,23 +8824,26 @@
vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data);
__ Add(data_reg, obj, Operand(data_offset));
- vixl::EmissionCheckScope guard(
- GetVIXLAssembler(),
- (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes);
- vixl32::Label return_address;
- EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address);
- __ cmp(mr, Operand(0));
- EmitPlaceholderBne(this, bne_label);
- ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset();
- __ ldr(ref_reg, MemOperand(data_reg, index_reg, vixl32::LSL, scale_factor));
- DCHECK(!needs_null_check); // The thunk cannot handle the null check.
- // Note: We need a Wide NEG for the unpoisoning.
- if (kPoisonHeapReferences) {
- __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0));
+ {
+ vixl::EmissionCheckScope guard(
+ GetVIXLAssembler(),
+ (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes);
+ vixl32::Label return_address;
+ EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address);
+ __ cmp(mr, Operand(0));
+ EmitPlaceholderBne(this, bne_label);
+ ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset();
+ __ ldr(ref_reg, MemOperand(data_reg, index_reg, vixl32::LSL, scale_factor));
+ DCHECK(!needs_null_check); // The thunk cannot handle the null check.
+ // Note: We need a Wide NEG for the unpoisoning.
+ if (kPoisonHeapReferences) {
+ __ rsb(EncodingSize(Wide), ref_reg, ref_reg, Operand(0));
+ }
+ __ Bind(&return_address);
+ DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(),
+ BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET);
}
- __ Bind(&return_address);
- DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(),
- BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET);
+ MaybeGenerateMarkingRegisterCheck(/* code */ 20, /* temp_loc */ LocationFrom(ip));
return;
}
@@ -8866,6 +8897,7 @@
// Fast path: the GC is not marking: just load the reference.
GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check);
__ Bind(slow_path->GetExitLabel());
+ MaybeGenerateMarkingRegisterCheck(/* code */ 21);
}
void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction,
@@ -8920,6 +8952,7 @@
// Fast path: the GC is not marking: nothing to do (the field is
// up-to-date, and we don't need to load the reference).
__ Bind(slow_path->GetExitLabel());
+ MaybeGenerateMarkingRegisterCheck(/* code */ 22);
}
void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction,
@@ -8981,6 +9014,20 @@
GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
}
+void CodeGeneratorARMVIXL::MaybeGenerateMarkingRegisterCheck(int code, Location temp_loc) {
+ // The following condition is a compile-time one, so it does not have a run-time cost.
+ if (kEmitCompilerReadBarrier && kUseBakerReadBarrier && kIsDebugBuild) {
+ // The following condition is a run-time one; it is executed after the
+ // previous compile-time test, to avoid penalizing non-debug builds.
+ if (GetCompilerOptions().EmitRunTimeChecksInDebugMode()) {
+ UseScratchRegisterScope temps(GetVIXLAssembler());
+ vixl32::Register temp = temp_loc.IsValid() ? RegisterFrom(temp_loc) : temps.Acquire();
+ GetAssembler()->GenerateMarkingRegisterCheck(temp,
+ kMarkingRegisterCheckBreakCodeBaseCode + code);
+ }
+ }
+}
+
void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction,
Location out,
Location ref,
diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h
index 7ab2993..5feb33b 100644
--- a/compiler/optimizing/code_generator_arm_vixl.h
+++ b/compiler/optimizing/code_generator_arm_vixl.h
@@ -661,6 +661,28 @@
ScaleFactor scale_factor,
bool needs_null_check);
+ // Emit code checking the status of the Marking Register, and
+ // aborting the program if MR does not match the value stored in the
+ // art::Thread object. Code is only emitted in debug mode and if
+ // CompilerOptions::EmitRunTimeChecksInDebugMode returns true.
+ //
+ // Argument `code` is used to identify the different occurrences of
+ // MaybeGenerateMarkingRegisterCheck in the code generator, and is
+ // used together with kMarkingRegisterCheckBreakCodeBaseCode to
+ // create the value passed to the BKPT instruction. Note that unlike
+ // in the ARM64 code generator, where `__LINE__` is passed as `code`
+ // argument to
+ // CodeGeneratorARM64::MaybeGenerateMarkingRegisterCheck, we cannot
+ // realistically do that here, as Encoding T1 for the BKPT
+ // instruction only accepts 8-bit immediate values.
+ //
+ // If `temp_loc` is a valid location, it is expected to be a
+ // register and will be used as a temporary to generate code;
+ // otherwise, a temporary will be fetched from the core register
+ // scratch pool.
+ virtual void MaybeGenerateMarkingRegisterCheck(int code,
+ Location temp_loc = Location::NoLocation());
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
diff --git a/compiler/optimizing/code_generator_vector_arm64.cc b/compiler/optimizing/code_generator_vector_arm64.cc
index f422b9f..9095ecd 100644
--- a/compiler/optimizing/code_generator_vector_arm64.cc
+++ b/compiler/optimizing/code_generator_vector_arm64.cc
@@ -15,7 +15,9 @@
*/
#include "code_generator_arm64.h"
+
#include "mirror/array-inl.h"
+#include "mirror/string.h"
using namespace vixl::aarch64; // NOLINT(build/namespaces)
diff --git a/compiler/optimizing/code_generator_vector_x86.cc b/compiler/optimizing/code_generator_vector_x86.cc
index 14782d7..e7aec76 100644
--- a/compiler/optimizing/code_generator_vector_x86.cc
+++ b/compiler/optimizing/code_generator_vector_x86.cc
@@ -15,7 +15,9 @@
*/
#include "code_generator_x86.h"
+
#include "mirror/array-inl.h"
+#include "mirror/string.h"
namespace art {
namespace x86 {
diff --git a/compiler/optimizing/code_generator_vector_x86_64.cc b/compiler/optimizing/code_generator_vector_x86_64.cc
index 246044e..c7ee81c 100644
--- a/compiler/optimizing/code_generator_vector_x86_64.cc
+++ b/compiler/optimizing/code_generator_vector_x86_64.cc
@@ -15,7 +15,9 @@
*/
#include "code_generator_x86_64.h"
+
#include "mirror/array-inl.h"
+#include "mirror/string.h"
namespace art {
namespace x86_64 {
diff --git a/compiler/optimizing/codegen_test_utils.h b/compiler/optimizing/codegen_test_utils.h
index cada2e6..aa4f5da 100644
--- a/compiler/optimizing/codegen_test_utils.h
+++ b/compiler/optimizing/codegen_test_utils.h
@@ -79,6 +79,21 @@
};
#ifdef ART_ENABLE_CODEGEN_arm
+// Special ARM code generator for codegen testing in a limited code
+// generation environment (i.e. with no runtime support).
+//
+// Note: If we want to exercise certains HIR constructions
+// (e.g. reference field load in Baker read barrier configuration) in
+// codegen tests in the future, we should also:
+// - save the Thread Register (R9) and possibly the Marking Register
+// (R8) before entering the generated function (both registers are
+// callee-save in AAPCS);
+// - set these registers to meaningful values before or upon entering
+// the generated function (so that generated code using them is
+// correct);
+// - restore their original values before leaving the generated
+// function.
+
// Provide our own codegen, that ensures the C calling conventions
// are preserved. Currently, ART and C do not match as R4 is caller-save
// in ART, and callee-save in C. Alternatively, we could use or write
@@ -100,6 +115,50 @@
blocked_core_registers_[arm::R6] = false;
blocked_core_registers_[arm::R7] = false;
}
+
+ void MaybeGenerateMarkingRegisterCheck(int code ATTRIBUTE_UNUSED,
+ Location temp_loc ATTRIBUTE_UNUSED) OVERRIDE {
+ // When turned on, the marking register checks in
+ // CodeGeneratorARMVIXL::MaybeGenerateMarkingRegisterCheck expects the
+ // Thread Register and the Marking Register to be set to
+ // meaningful values. This is not the case in codegen testing, so
+ // just disable them entirely here (by doing nothing in this
+ // method).
+ }
+};
+#endif
+
+#ifdef ART_ENABLE_CODEGEN_arm64
+// Special ARM64 code generator for codegen testing in a limited code
+// generation environment (i.e. with no runtime support).
+//
+// Note: If we want to exercise certains HIR constructions
+// (e.g. reference field load in Baker read barrier configuration) in
+// codegen tests in the future, we should also:
+// - save the Thread Register (X19) and possibly the Marking Register
+// (X20) before entering the generated function (both registers are
+// callee-save in AAPCS64);
+// - set these registers to meaningful values before or upon entering
+// the generated function (so that generated code using them is
+// correct);
+// - restore their original values before leaving the generated
+// function.
+class TestCodeGeneratorARM64 : public arm64::CodeGeneratorARM64 {
+ public:
+ TestCodeGeneratorARM64(HGraph* graph,
+ const Arm64InstructionSetFeatures& isa_features,
+ const CompilerOptions& compiler_options)
+ : arm64::CodeGeneratorARM64(graph, isa_features, compiler_options) {}
+
+ void MaybeGenerateMarkingRegisterCheck(int codem ATTRIBUTE_UNUSED,
+ Location temp_loc ATTRIBUTE_UNUSED) OVERRIDE {
+ // When turned on, the marking register checks in
+ // CodeGeneratorARM64::MaybeGenerateMarkingRegisterCheck expect the
+ // Thread Register and the Marking Register to be set to
+ // meaningful values. This is not the case in codegen testing, so
+ // just disable them entirely here (by doing nothing in this
+ // method).
+ }
};
#endif
@@ -263,7 +322,8 @@
bool has_result,
Expected expected) {
CompilerOptions compiler_options;
- std::unique_ptr<CodeGenerator> codegen(target_config.CreateCodeGenerator(graph, compiler_options));
+ std::unique_ptr<CodeGenerator> codegen(target_config.CreateCodeGenerator(graph,
+ compiler_options));
RunCode(codegen.get(), graph, hook_before_codegen, has_result, expected);
}
@@ -280,9 +340,8 @@
CodeGenerator* create_codegen_arm64(HGraph* graph, const CompilerOptions& compiler_options) {
std::unique_ptr<const Arm64InstructionSetFeatures> features_arm64(
Arm64InstructionSetFeatures::FromCppDefines());
- return new (graph->GetArena()) arm64::CodeGeneratorARM64(graph,
- *features_arm64.get(),
- compiler_options);
+ return new (graph->GetArena())
+ TestCodeGeneratorARM64(graph, *features_arm64.get(), compiler_options);
}
#endif
diff --git a/compiler/optimizing/induction_var_range.h b/compiler/optimizing/induction_var_range.h
index ab1772b..0b980f5 100644
--- a/compiler/optimizing/induction_var_range.h
+++ b/compiler/optimizing/induction_var_range.h
@@ -151,6 +151,16 @@
}
/**
+ * Checks if the given phi instruction has been classified as anything by
+ * induction variable analysis. Returns false for anything that cannot be
+ * classified statically, such as reductions or other complex cycles.
+ */
+ bool IsClassified(HPhi* phi) const {
+ HLoopInformation* lp = phi->GetBlock()->GetLoopInformation(); // closest enveloping loop
+ return (lp != nullptr) && (induction_analysis_->LookupInfo(lp, phi) != nullptr);
+ }
+
+ /**
* Checks if header logic of a loop terminates. Sets trip-count tc if known.
*/
bool IsFinite(HLoopInformation* loop, /*out*/ int64_t* tc) const;
diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc
index 4cea6df..2669d97 100644
--- a/compiler/optimizing/intrinsics_mips.cc
+++ b/compiler/optimizing/intrinsics_mips.cc
@@ -22,6 +22,7 @@
#include "entrypoints/quick/quick_entrypoints.h"
#include "intrinsics.h"
#include "mirror/array-inl.h"
+#include "mirror/object_array-inl.h"
#include "mirror/string.h"
#include "scoped_thread_state_change-inl.h"
#include "thread.h"
diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc
index d785567..74be954 100644
--- a/compiler/optimizing/intrinsics_mips64.cc
+++ b/compiler/optimizing/intrinsics_mips64.cc
@@ -22,6 +22,7 @@
#include "entrypoints/quick/quick_entrypoints.h"
#include "intrinsics.h"
#include "mirror/array-inl.h"
+#include "mirror/object_array-inl.h"
#include "mirror/string.h"
#include "scoped_thread_state_change-inl.h"
#include "thread.h"
diff --git a/compiler/optimizing/load_store_analysis.h b/compiler/optimizing/load_store_analysis.h
index 86fb8e0..a2c1794 100644
--- a/compiler/optimizing/load_store_analysis.h
+++ b/compiler/optimizing/load_store_analysis.h
@@ -466,6 +466,11 @@
CreateReferenceInfoForReferenceType(new_instance);
}
+ void VisitNewArray(HNewArray* new_array) OVERRIDE {
+ // Any references appearing in the ref_info_array_ so far cannot alias with new_array.
+ CreateReferenceInfoForReferenceType(new_array);
+ }
+
void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* instruction) OVERRIDE {
CreateReferenceInfoForReferenceType(instruction);
}
@@ -478,6 +483,22 @@
CreateReferenceInfoForReferenceType(instruction);
}
+ void VisitInvokeUnresolved(HInvokeUnresolved* instruction) OVERRIDE {
+ CreateReferenceInfoForReferenceType(instruction);
+ }
+
+ void VisitInvokePolymorphic(HInvokePolymorphic* instruction) OVERRIDE {
+ CreateReferenceInfoForReferenceType(instruction);
+ }
+
+ void VisitLoadString(HLoadString* instruction) OVERRIDE {
+ CreateReferenceInfoForReferenceType(instruction);
+ }
+
+ void VisitPhi(HPhi* instruction) OVERRIDE {
+ CreateReferenceInfoForReferenceType(instruction);
+ }
+
void VisitParameterValue(HParameterValue* instruction) OVERRIDE {
CreateReferenceInfoForReferenceType(instruction);
}
diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc
index 0ef7dcd..027ba77 100644
--- a/compiler/optimizing/loop_optimization.cc
+++ b/compiler/optimizing/loop_optimization.cc
@@ -256,6 +256,35 @@
return false;
}
+// Detect reductions of the following forms,
+// under assumption phi has only *one* use:
+// x = x_phi + ..
+// x = x_phi - ..
+// x = max(x_phi, ..)
+// x = min(x_phi, ..)
+static bool HasReductionFormat(HInstruction* reduction, HInstruction* phi) {
+ if (reduction->IsAdd()) {
+ return reduction->InputAt(0) == phi || reduction->InputAt(1) == phi;
+ } else if (reduction->IsSub()) {
+ return reduction->InputAt(0) == phi;
+ } else if (reduction->IsInvokeStaticOrDirect()) {
+ switch (reduction->AsInvokeStaticOrDirect()->GetIntrinsic()) {
+ case Intrinsics::kMathMinIntInt:
+ case Intrinsics::kMathMinLongLong:
+ case Intrinsics::kMathMinFloatFloat:
+ case Intrinsics::kMathMinDoubleDouble:
+ case Intrinsics::kMathMaxIntInt:
+ case Intrinsics::kMathMaxLongLong:
+ case Intrinsics::kMathMaxFloatFloat:
+ case Intrinsics::kMathMaxDoubleDouble:
+ return reduction->InputAt(0) == phi || reduction->InputAt(1) == phi;
+ default:
+ return false;
+ }
+ }
+ return false;
+}
+
// Test vector restrictions.
static bool HasVectorRestrictions(uint64_t restrictions, uint64_t tested) {
return (restrictions & tested) != 0;
@@ -280,12 +309,11 @@
return false;
}
}
-
return true;
}
//
-// Class methods.
+// Public methods.
//
HLoopOptimization::HLoopOptimization(HGraph* graph,
@@ -299,7 +327,7 @@
top_loop_(nullptr),
last_loop_(nullptr),
iset_(nullptr),
- induction_simplication_count_(0),
+ reductions_(nullptr),
simplified_(false),
vector_length_(0),
vector_refs_(nullptr),
@@ -333,6 +361,10 @@
last_loop_ = top_loop_ = nullptr;
}
+//
+// Loop setup and traversal.
+//
+
void HLoopOptimization::LocalRun() {
// Build the linear order using the phase-local allocator. This step enables building
// a loop hierarchy that properly reflects the outer-inner and previous-next relation.
@@ -351,17 +383,21 @@
// should use the global allocator.
if (top_loop_ != nullptr) {
ArenaSet<HInstruction*> iset(loop_allocator_->Adapter(kArenaAllocLoopOptimization));
+ ArenaSafeMap<HInstruction*, HInstruction*> reds(
+ std::less<HInstruction*>(), loop_allocator_->Adapter(kArenaAllocLoopOptimization));
ArenaSet<ArrayReference> refs(loop_allocator_->Adapter(kArenaAllocLoopOptimization));
ArenaSafeMap<HInstruction*, HInstruction*> map(
std::less<HInstruction*>(), loop_allocator_->Adapter(kArenaAllocLoopOptimization));
// Attach.
iset_ = &iset;
+ reductions_ = &reds;
vector_refs_ = &refs;
vector_map_ = ↦
// Traverse.
TraverseLoopsInnerToOuter(top_loop_);
// Detach.
iset_ = nullptr;
+ reductions_ = nullptr;
vector_refs_ = nullptr;
vector_map_ = nullptr;
}
@@ -414,16 +450,12 @@
}
}
-void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) {
+bool HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) {
+ bool changed = false;
for ( ; node != nullptr; node = node->next) {
- // Visit inner loops first.
- uint32_t current_induction_simplification_count = induction_simplication_count_;
- if (node->inner != nullptr) {
- TraverseLoopsInnerToOuter(node->inner);
- }
- // Recompute induction information of this loop if the induction
- // of any inner loop has been simplified.
- if (current_induction_simplification_count != induction_simplication_count_) {
+ // Visit inner loops first. Recompute induction information for this
+ // loop if the induction of any inner loop has changed.
+ if (TraverseLoopsInnerToOuter(node->inner)) {
induction_range_.ReVisit(node->loop_info);
}
// Repeat simplifications in the loop-body until no more changes occur.
@@ -433,12 +465,14 @@
simplified_ = false;
SimplifyInduction(node);
SimplifyBlocks(node);
+ changed = simplified_ || changed;
} while (simplified_);
// Optimize inner loop.
if (node->inner == nullptr) {
- OptimizeInnerLoop(node);
+ changed = OptimizeInnerLoop(node) || changed;
}
}
+ return changed;
}
//
@@ -455,20 +489,18 @@
// for (int i = 0; i < 10; i++, k++) { .... no k .... } return k;
for (HInstructionIterator it(header->GetPhis()); !it.Done(); it.Advance()) {
HPhi* phi = it.Current()->AsPhi();
- iset_->clear(); // prepare phi induction
if (TrySetPhiInduction(phi, /*restrict_uses*/ true) &&
TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ false)) {
// Note that it's ok to have replaced uses after the loop with the last value, without
// being able to remove the cycle. Environment uses (which are the reason we may not be
- // able to remove the cycle) within the loop will still hold the right value.
+ // able to remove the cycle) within the loop will still hold the right value. We must
+ // have tried first, however, to replace outside uses.
if (CanRemoveCycle()) {
+ simplified_ = true;
for (HInstruction* i : *iset_) {
RemoveFromCycle(i);
}
-
- // Check that there are no records of the deleted instructions.
DCHECK(CheckInductionSetFullyRemoved(iset_));
- simplified_ = true;
}
}
}
@@ -511,21 +543,20 @@
}
}
-void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) {
+bool HLoopOptimization::OptimizeInnerLoop(LoopNode* node) {
HBasicBlock* header = node->loop_info->GetHeader();
HBasicBlock* preheader = node->loop_info->GetPreHeader();
// Ensure loop header logic is finite.
int64_t trip_count = 0;
if (!induction_range_.IsFinite(node->loop_info, &trip_count)) {
- return;
+ return false;
}
-
// Ensure there is only a single loop-body (besides the header).
HBasicBlock* body = nullptr;
for (HBlocksInLoopIterator it(*node->loop_info); !it.Done(); it.Advance()) {
if (it.Current() != header) {
if (body != nullptr) {
- return;
+ return false;
}
body = it.Current();
}
@@ -533,27 +564,27 @@
CHECK(body != nullptr);
// Ensure there is only a single exit point.
if (header->GetSuccessors().size() != 2) {
- return;
+ return false;
}
HBasicBlock* exit = (header->GetSuccessors()[0] == body)
? header->GetSuccessors()[1]
: header->GetSuccessors()[0];
// Ensure exit can only be reached by exiting loop.
if (exit->GetPredecessors().size() != 1) {
- return;
+ return false;
}
// Detect either an empty loop (no side effects other than plain iteration) or
// a trivial loop (just iterating once). Replace subsequent index uses, if any,
// with the last value and remove the loop, possibly after unrolling its body.
- HInstruction* phi = header->GetFirstPhi();
- iset_->clear(); // prepare phi induction
- if (TrySetSimpleLoopHeader(header)) {
+ HPhi* main_phi = nullptr;
+ if (TrySetSimpleLoopHeader(header, &main_phi)) {
bool is_empty = IsEmptyBody(body);
- if ((is_empty || trip_count == 1) &&
- TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ true)) {
+ if (reductions_->empty() && // TODO: possible with some effort
+ (is_empty || trip_count == 1) &&
+ TryAssignLastValue(node->loop_info, main_phi, preheader, /*collect_loop_uses*/ true)) {
if (!is_empty) {
// Unroll the loop-body, which sees initial value of the index.
- phi->ReplaceWith(phi->InputAt(0));
+ main_phi->ReplaceWith(main_phi->InputAt(0));
preheader->MergeInstructionsWith(body);
}
body->DisconnectAndDelete();
@@ -566,21 +597,20 @@
preheader->AddDominatedBlock(exit);
exit->SetDominator(preheader);
RemoveLoop(node); // update hierarchy
- return;
+ return true;
}
}
-
// Vectorize loop, if possible and valid.
- if (kEnableVectorization) {
- iset_->clear(); // prepare phi induction
- if (TrySetSimpleLoopHeader(header) &&
- ShouldVectorize(node, body, trip_count) &&
- TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ true)) {
- Vectorize(node, body, exit, trip_count);
- graph_->SetHasSIMD(true); // flag SIMD usage
- return;
- }
+ if (kEnableVectorization &&
+ TrySetSimpleLoopHeader(header, &main_phi) &&
+ reductions_->empty() && // TODO: possible with some effort
+ ShouldVectorize(node, body, trip_count) &&
+ TryAssignLastValue(node->loop_info, main_phi, preheader, /*collect_loop_uses*/ true)) {
+ Vectorize(node, body, exit, trip_count);
+ graph_->SetHasSIMD(true); // flag SIMD usage
+ return true;
}
+ return false;
}
//
@@ -621,6 +651,8 @@
// aliased, as well as the property that references either point to the same
// array or to two completely disjoint arrays, i.e., no partial aliasing.
// Other than a few simply heuristics, no detailed subscript analysis is done.
+ // The scan over references also finds a suitable dynamic loop peeling candidate.
+ const ArrayReference* candidate = nullptr;
for (auto i = vector_refs_->begin(); i != vector_refs_->end(); ++i) {
for (auto j = i; ++j != vector_refs_->end(); ) {
if (i->type == j->type && (i->lhs || j->lhs)) {
@@ -656,7 +688,7 @@
}
// Consider dynamic loop peeling for alignment.
- SetPeelingCandidate(trip_count);
+ SetPeelingCandidate(candidate, trip_count);
// Success!
return true;
@@ -679,14 +711,15 @@
bool needs_cleanup = trip_count == 0 || (trip_count % chunk) != 0;
// Adjust vector bookkeeping.
- iset_->clear(); // prepare phi induction
- bool is_simple_loop_header = TrySetSimpleLoopHeader(header); // fills iset_
+ HPhi* main_phi = nullptr;
+ bool is_simple_loop_header = TrySetSimpleLoopHeader(header, &main_phi); // refills sets
DCHECK(is_simple_loop_header);
vector_header_ = header;
vector_body_ = block;
- // Generate dynamic loop peeling trip count, if needed:
- // ptc = <peeling-needed-for-candidate>
+ // Generate dynamic loop peeling trip count, if needed, under the assumption
+ // that the Android runtime guarantees at least "component size" alignment:
+ // ptc = (ALIGN - (&a[initial] % ALIGN)) / type-size
HInstruction* ptc = nullptr;
if (vector_peeling_candidate_ != nullptr) {
DCHECK_LT(vector_length_, trip_count) << "dynamic peeling currently requires known trip count";
@@ -775,6 +808,7 @@
while (!header->GetFirstInstruction()->IsGoto()) {
header->RemoveInstruction(header->GetFirstInstruction());
}
+
// Update loop hierarchy: the old header now resides in the same outer loop
// as the old preheader. Note that we don't bother putting sequential
// loops back in the hierarchy at this point.
@@ -841,13 +875,12 @@
vector_index_ = new (global_allocator_) HAdd(induc_type, vector_index_, step);
Insert(vector_body_, vector_index_);
}
- // Finalize phi for the loop index.
+ // Finalize phi inputs for the loop index.
phi->AddInput(lo);
phi->AddInput(vector_index_);
vector_index_ = phi;
}
-// TODO: accept reductions at left-hand-side, mixed-type store idioms, etc.
bool HLoopOptimization::VectorizeDef(LoopNode* node,
HInstruction* instruction,
bool generate_code) {
@@ -1118,7 +1151,7 @@
case kArm:
case kThumb2:
// Allow vectorization for all ARM devices, because Android assumes that
- // ARM 32-bit always supports advanced SIMD.
+ // ARM 32-bit always supports advanced SIMD (64-bit SIMD).
switch (type) {
case Primitive::kPrimBoolean:
case Primitive::kPrimByte:
@@ -1137,7 +1170,7 @@
return false;
case kArm64:
// Allow vectorization for all ARM devices, because Android assumes that
- // ARMv8 AArch64 always supports advanced SIMD.
+ // ARMv8 AArch64 always supports advanced SIMD (128-bit SIMD).
switch (type) {
case Primitive::kPrimBoolean:
case Primitive::kPrimByte:
@@ -1162,7 +1195,7 @@
}
case kX86:
case kX86_64:
- // Allow vectorization for SSE4-enabled X86 devices only (128-bit vectors).
+ // Allow vectorization for SSE4.1-enabled X86 devices only (128-bit SIMD).
if (features->AsX86InstructionSetFeatures()->HasSSE4_1()) {
switch (type) {
case Primitive::kPrimBoolean:
@@ -1310,8 +1343,6 @@
global_allocator_, base, opa, type, vector_length_, is_string_char_at);
}
// Known dynamically enforced alignment?
- // TODO: detect offset + constant differences.
- // TODO: long run, static alignment analysis?
if (vector_peeling_candidate_ != nullptr &&
vector_peeling_candidate_->base == base &&
vector_peeling_candidate_->offset == offset) {
@@ -1586,9 +1617,11 @@
return true;
}
-void HLoopOptimization::SetPeelingCandidate(int64_t trip_count ATTRIBUTE_UNUSED) {
+void HLoopOptimization::SetPeelingCandidate(const ArrayReference* candidate,
+ int64_t trip_count ATTRIBUTE_UNUSED) {
// Current heuristic: none.
// TODO: implement
+ vector_peeling_candidate_ = candidate;
}
uint32_t HLoopOptimization::GetUnrollingFactor(HBasicBlock* block, int64_t trip_count) {
@@ -1616,13 +1649,17 @@
//
bool HLoopOptimization::TrySetPhiInduction(HPhi* phi, bool restrict_uses) {
+ // Start with empty phi induction.
+ iset_->clear();
+
// Special case Phis that have equivalent in a debuggable setup. Our graph checker isn't
// smart enough to follow strongly connected components (and it's probably not worth
// it to make it so). See b/33775412.
if (graph_->IsDebuggable() && phi->HasEquivalentPhi()) {
return false;
}
- DCHECK(iset_->empty());
+
+ // Lookup phi induction cycle.
ArenaSet<HInstruction*>* set = induction_range_.LookupCycle(phi);
if (set != nullptr) {
for (HInstruction* i : *set) {
@@ -1634,6 +1671,7 @@
} else if (!i->IsRemovable()) {
return false;
} else if (i != phi && restrict_uses) {
+ // Deal with regular uses.
for (const HUseListNode<HInstruction*>& use : i->GetUses()) {
if (set->find(use.GetUser()) == set->end()) {
return false;
@@ -1647,17 +1685,65 @@
return false;
}
-// Find: phi: Phi(init, addsub)
-// s: SuspendCheck
-// c: Condition(phi, bound)
-// i: If(c)
-// TODO: Find a less pattern matching approach?
-bool HLoopOptimization::TrySetSimpleLoopHeader(HBasicBlock* block) {
+bool HLoopOptimization::TrySetPhiReduction(HPhi* phi) {
DCHECK(iset_->empty());
- HInstruction* phi = block->GetFirstPhi();
- if (phi != nullptr &&
- phi->GetNext() == nullptr &&
- TrySetPhiInduction(phi->AsPhi(), /*restrict_uses*/ false)) {
+ // Only unclassified phi cycles are candidates for reductions.
+ if (induction_range_.IsClassified(phi)) {
+ return false;
+ }
+ // Accept operations like x = x + .., provided that the phi and the reduction are
+ // used exactly once inside the loop, and by each other.
+ HInputsRef inputs = phi->GetInputs();
+ if (inputs.size() == 2) {
+ HInstruction* reduction = inputs[1];
+ if (HasReductionFormat(reduction, phi)) {
+ HLoopInformation* loop_info = phi->GetBlock()->GetLoopInformation();
+ int32_t use_count = 0;
+ bool single_use_inside_loop =
+ // Reduction update only used by phi.
+ reduction->GetUses().HasExactlyOneElement() &&
+ !reduction->HasEnvironmentUses() &&
+ // Reduction update is only use of phi inside the loop.
+ IsOnlyUsedAfterLoop(loop_info, phi, /*collect_loop_uses*/ true, &use_count) &&
+ iset_->size() == 1;
+ iset_->clear(); // leave the way you found it
+ if (single_use_inside_loop) {
+ // Link reduction back, and start recording feed value.
+ reductions_->Put(reduction, phi);
+ reductions_->Put(phi, phi->InputAt(0));
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+bool HLoopOptimization::TrySetSimpleLoopHeader(HBasicBlock* block, /*out*/ HPhi** main_phi) {
+ // Start with empty phi induction and reductions.
+ iset_->clear();
+ reductions_->clear();
+
+ // Scan the phis to find the following (the induction structure has already
+ // been optimized, so we don't need to worry about trivial cases):
+ // (1) optional reductions in loop,
+ // (2) the main induction, used in loop control.
+ HPhi* phi = nullptr;
+ for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
+ if (TrySetPhiReduction(it.Current()->AsPhi())) {
+ continue;
+ } else if (phi == nullptr) {
+ // Found the first candidate for main induction.
+ phi = it.Current()->AsPhi();
+ } else {
+ return false;
+ }
+ }
+
+ // Then test for a typical loopheader:
+ // s: SuspendCheck
+ // c: Condition(phi, bound)
+ // i: If(c)
+ if (phi != nullptr && TrySetPhiInduction(phi, /*restrict_uses*/ false)) {
HInstruction* s = block->GetFirstInstruction();
if (s != nullptr && s->IsSuspendCheck()) {
HInstruction* c = s->GetNext();
@@ -1669,6 +1755,7 @@
if (i != nullptr && i->IsIf() && i->InputAt(0) == c) {
iset_->insert(c);
iset_->insert(s);
+ *main_phi = phi;
return true;
}
}
@@ -1692,6 +1779,7 @@
bool HLoopOptimization::IsUsedOutsideLoop(HLoopInformation* loop_info,
HInstruction* instruction) {
+ // Deal with regular uses.
for (const HUseListNode<HInstruction*>& use : instruction->GetUses()) {
if (use.GetUser()->GetBlock()->GetLoopInformation() != loop_info) {
return true;
@@ -1704,6 +1792,7 @@
HInstruction* instruction,
bool collect_loop_uses,
/*out*/ int32_t* use_count) {
+ // Deal with regular uses.
for (const HUseListNode<HInstruction*>& use : instruction->GetUses()) {
HInstruction* user = use.GetUser();
if (iset_->find(user) == iset_->end()) { // not excluded?
@@ -1729,6 +1818,7 @@
// Try to replace outside uses with the last value.
if (induction_range_.CanGenerateLastValue(instruction)) {
HInstruction* replacement = induction_range_.GenerateLastValue(instruction, graph_, block);
+ // Deal with regular uses.
const HUseList<HInstruction*>& uses = instruction->GetUses();
for (auto it = uses.begin(), end = uses.end(); it != end;) {
HInstruction* user = it->GetUser();
@@ -1744,6 +1834,7 @@
induction_range_.Replace(user, instruction, replacement); // update induction
}
}
+ // Deal with environment uses.
const HUseList<HEnvironment*>& env_uses = instruction->GetEnvUses();
for (auto it = env_uses.begin(), end = env_uses.end(); it != end;) {
HEnvironment* user = it->GetUser();
@@ -1759,7 +1850,6 @@
}
}
}
- induction_simplication_count_++;
return true;
}
return false;
diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h
index de4bd85..49be8a3 100644
--- a/compiler/optimizing/loop_optimization.h
+++ b/compiler/optimizing/loop_optimization.h
@@ -104,18 +104,33 @@
bool lhs; // def/use
};
+ //
// Loop setup and traversal.
+ //
+
void LocalRun();
void AddLoop(HLoopInformation* loop_info);
void RemoveLoop(LoopNode* node);
- void TraverseLoopsInnerToOuter(LoopNode* node);
+ // Traverses all loops inner to outer to perform simplifications and optimizations.
+ // Returns true if loops nested inside current loop (node) have changed.
+ bool TraverseLoopsInnerToOuter(LoopNode* node);
+
+ //
// Optimization.
+ //
+
void SimplifyInduction(LoopNode* node);
void SimplifyBlocks(LoopNode* node);
- void OptimizeInnerLoop(LoopNode* node);
+ // Performs optimizations specific to inner loop (empty loop removal,
+ // unrolling, vectorization). Returns true if anything changed.
+ bool OptimizeInnerLoop(LoopNode* node);
+
+ //
// Vectorization analysis and synthesis.
+ //
+
bool ShouldVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count);
void Vectorize(LoopNode* node, HBasicBlock* block, HBasicBlock* exit, int64_t trip_count);
void GenerateNewLoop(LoopNode* node,
@@ -155,12 +170,20 @@
// Vectorization heuristics.
bool IsVectorizationProfitable(int64_t trip_count);
- void SetPeelingCandidate(int64_t trip_count);
+ void SetPeelingCandidate(const ArrayReference* candidate, int64_t trip_count);
uint32_t GetUnrollingFactor(HBasicBlock* block, int64_t trip_count);
+ //
// Helpers.
+ //
+
bool TrySetPhiInduction(HPhi* phi, bool restrict_uses);
- bool TrySetSimpleLoopHeader(HBasicBlock* block);
+ bool TrySetPhiReduction(HPhi* phi);
+
+ // Detects loop header with a single induction (returned in main_phi), possibly
+ // other phis for reductions, but no other side effects. Returns true on success.
+ bool TrySetSimpleLoopHeader(HBasicBlock* block, /*out*/ HPhi** main_phi);
+
bool IsEmptyBody(HBasicBlock* block);
bool IsOnlyUsedAfterLoop(HLoopInformation* loop_info,
HInstruction* instruction,
@@ -200,10 +223,12 @@
// Contents reside in phase-local heap memory.
ArenaSet<HInstruction*>* iset_;
- // Counter that tracks how many induction cycles have been simplified. Useful
- // to trigger incremental updates of induction variable analysis of outer loops
- // when the induction of inner loops has changed.
- uint32_t induction_simplication_count_;
+ // Temporary bookkeeping of reduction instructions. Mapping is two-fold:
+ // (1) reductions in the loop-body are mapped back to their phi definition,
+ // (2) phi definitions are mapped to their initial value (updated during
+ // code generation to feed the proper values into the new chain).
+ // Contents reside in phase-local heap memory.
+ ArenaSafeMap<HInstruction*, HInstruction*>* reductions_;
// Flag that tracks if any simplifications have occurred.
bool simplified_;
diff --git a/compiler/optimizing/loop_optimization_test.cc b/compiler/optimizing/loop_optimization_test.cc
index 5b93506..b5b03d8 100644
--- a/compiler/optimizing/loop_optimization_test.cc
+++ b/compiler/optimizing/loop_optimization_test.cc
@@ -195,4 +195,44 @@
EXPECT_EQ("[[[[[[[[[[][][][][][][][][][]]]]]]]]]]", LoopStructure());
}
+// Check that SimplifyLoop() doesn't invalidate data flow when ordering loop headers'
+// predecessors.
+TEST_F(LoopOptimizationTest, SimplifyLoop) {
+ // Can't use AddLoop as we want special order for blocks predecessors.
+ HBasicBlock* header = new (&allocator_) HBasicBlock(graph_);
+ HBasicBlock* body = new (&allocator_) HBasicBlock(graph_);
+ graph_->AddBlock(header);
+ graph_->AddBlock(body);
+
+ // Control flow: make a loop back edge first in the list of predecessors.
+ entry_block_->RemoveSuccessor(return_block_);
+ body->AddSuccessor(header);
+ entry_block_->AddSuccessor(header);
+ header->AddSuccessor(body);
+ header->AddSuccessor(return_block_);
+ DCHECK(header->GetSuccessors()[1] == return_block_);
+
+ // Data flow.
+ header->AddInstruction(new (&allocator_) HIf(parameter_));
+ body->AddInstruction(new (&allocator_) HGoto());
+
+ HPhi* phi = new (&allocator_) HPhi(&allocator_, 0, 0, Primitive::kPrimInt);
+ HInstruction* add = new (&allocator_) HAdd(Primitive::kPrimInt, phi, parameter_);
+ header->AddPhi(phi);
+ body->AddInstruction(add);
+
+ phi->AddInput(add);
+ phi->AddInput(parameter_);
+
+ graph_->ClearLoopInformation();
+ graph_->ClearDominanceInformation();
+ graph_->BuildDominatorTree();
+
+ // Check that after optimizations in BuildDominatorTree()/SimplifyCFG() phi inputs
+ // are still mapped correctly to the block predecessors.
+ for (size_t i = 0, e = phi->InputCount(); i < e; i++) {
+ HInstruction* input = phi->InputAt(i);
+ ASSERT_TRUE(input->GetBlock()->Dominates(header->GetPredecessors()[i]));
+ }
+}
} // namespace art
diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index 8644f67..508027e 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc
@@ -358,6 +358,35 @@
}
}
+// Reorder phi inputs to match reordering of the block's predecessors.
+static void FixPhisAfterPredecessorsReodering(HBasicBlock* block, size_t first, size_t second) {
+ for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
+ HPhi* phi = it.Current()->AsPhi();
+ HInstruction* first_instr = phi->InputAt(first);
+ HInstruction* second_instr = phi->InputAt(second);
+ phi->ReplaceInput(first_instr, second);
+ phi->ReplaceInput(second_instr, first);
+ }
+}
+
+// Make sure that the first predecessor of a loop header is the incoming block.
+void HGraph::OrderLoopHeaderPredecessors(HBasicBlock* header) {
+ DCHECK(header->IsLoopHeader());
+ HLoopInformation* info = header->GetLoopInformation();
+ if (info->IsBackEdge(*header->GetPredecessors()[0])) {
+ HBasicBlock* to_swap = header->GetPredecessors()[0];
+ for (size_t pred = 1, e = header->GetPredecessors().size(); pred < e; ++pred) {
+ HBasicBlock* predecessor = header->GetPredecessors()[pred];
+ if (!info->IsBackEdge(*predecessor)) {
+ header->predecessors_[pred] = to_swap;
+ header->predecessors_[0] = predecessor;
+ FixPhisAfterPredecessorsReodering(header, 0, pred);
+ break;
+ }
+ }
+ }
+}
+
void HGraph::SimplifyLoop(HBasicBlock* header) {
HLoopInformation* info = header->GetLoopInformation();
@@ -381,18 +410,7 @@
pre_header->AddSuccessor(header);
}
- // Make sure the first predecessor of a loop header is the incoming block.
- if (info->IsBackEdge(*header->GetPredecessors()[0])) {
- HBasicBlock* to_swap = header->GetPredecessors()[0];
- for (size_t pred = 1, e = header->GetPredecessors().size(); pred < e; ++pred) {
- HBasicBlock* predecessor = header->GetPredecessors()[pred];
- if (!info->IsBackEdge(*predecessor)) {
- header->predecessors_[pred] = to_swap;
- header->predecessors_[0] = predecessor;
- break;
- }
- }
- }
+ OrderLoopHeaderPredecessors(header);
HInstruction* first_instruction = header->GetFirstInstruction();
if (first_instruction != nullptr && first_instruction->IsSuspendCheck()) {
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index 29be8ac..68d6c2e 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -418,6 +418,7 @@
HBasicBlock* SplitEdge(HBasicBlock* block, HBasicBlock* successor);
void SplitCriticalEdge(HBasicBlock* block, HBasicBlock* successor);
+ void OrderLoopHeaderPredecessors(HBasicBlock* header);
void SimplifyLoop(HBasicBlock* header);
int32_t GetNextInstructionId() {
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index 435ca1c..426a169 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -22,8 +22,6 @@
#include <stdint.h>
-#include "android-base/strings.h"
-
#ifdef ART_ENABLE_CODEGEN_arm64
#include "instruction_simplifier_arm64.h"
#endif
@@ -1127,12 +1125,7 @@
bool IsCompilingWithCoreImage() {
const std::string& image = Runtime::Current()->GetImageLocation();
- // TODO: This is under-approximating...
- if (android::base::EndsWith(image, "core.art") ||
- android::base::EndsWith(image, "core-optimizing.art")) {
- return true;
- }
- return false;
+ return CompilerDriver::IsCoreImageFilename(image);
}
bool EncodeArtMethodInInlineInfo(ArtMethod* method ATTRIBUTE_UNUSED) {
@@ -1226,14 +1219,14 @@
uint8_t* stack_map_data = nullptr;
uint8_t* method_info_data = nullptr;
uint8_t* roots_data = nullptr;
- code_cache->ReserveData(self,
- stack_map_size,
- method_info_size,
- number_of_roots,
- method,
- &stack_map_data,
- &method_info_data,
- &roots_data);
+ uint32_t data_size = code_cache->ReserveData(self,
+ stack_map_size,
+ method_info_size,
+ number_of_roots,
+ method,
+ &stack_map_data,
+ &method_info_data,
+ &roots_data);
if (stack_map_data == nullptr || roots_data == nullptr) {
return false;
}
@@ -1254,6 +1247,7 @@
codegen->GetFpuSpillMask(),
code_allocator.GetMemory().data(),
code_allocator.GetSize(),
+ data_size,
osr,
roots,
codegen->GetGraph()->HasShouldDeoptimizeFlag(),
diff --git a/compiler/optimizing/scheduler.cc b/compiler/optimizing/scheduler.cc
index 5ad011d..38cd51b 100644
--- a/compiler/optimizing/scheduler.cc
+++ b/compiler/optimizing/scheduler.cc
@@ -554,6 +554,14 @@
}
void HScheduler::Schedule(HGraph* graph) {
+ // We run lsa here instead of in a separate pass to better control whether we
+ // should run the analysis or not.
+ LoadStoreAnalysis lsa(graph);
+ if (!only_optimize_loop_blocks_ || graph->HasLoops()) {
+ lsa.Run();
+ scheduling_graph_.SetHeapLocationCollector(lsa.GetHeapLocationCollector());
+ }
+
for (HBasicBlock* block : graph->GetReversePostOrder()) {
if (IsSchedulable(block)) {
Schedule(block);
@@ -566,14 +574,6 @@
// Build the scheduling graph.
scheduling_graph_.Clear();
-
- // Only perform LSA/HeapLocation analysis on the basic block that
- // is going to get instruction scheduled.
- HeapLocationCollector heap_location_collector(block->GetGraph());
- heap_location_collector.VisitBasicBlock(block);
- heap_location_collector.BuildAliasingMatrix();
- scheduling_graph_.SetHeapLocationCollector(heap_location_collector);
-
for (HBackwardInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
HInstruction* instruction = it.Current();
CHECK_EQ(instruction->GetBlock(), block)
@@ -724,8 +724,8 @@
instruction->IsClassTableGet() ||
instruction->IsCurrentMethod() ||
instruction->IsDivZeroCheck() ||
- instruction->IsInstanceFieldGet() ||
- instruction->IsInstanceFieldSet() ||
+ (instruction->IsInstanceFieldGet() && !instruction->AsInstanceFieldGet()->IsVolatile()) ||
+ (instruction->IsInstanceFieldSet() && !instruction->AsInstanceFieldSet()->IsVolatile()) ||
instruction->IsInstanceOf() ||
instruction->IsInvokeInterface() ||
instruction->IsInvokeStaticOrDirect() ||
@@ -741,14 +741,10 @@
instruction->IsReturn() ||
instruction->IsReturnVoid() ||
instruction->IsSelect() ||
- instruction->IsStaticFieldGet() ||
- instruction->IsStaticFieldSet() ||
+ (instruction->IsStaticFieldGet() && !instruction->AsStaticFieldGet()->IsVolatile()) ||
+ (instruction->IsStaticFieldSet() && !instruction->AsStaticFieldSet()->IsVolatile()) ||
instruction->IsSuspendCheck() ||
- instruction->IsTypeConversion() ||
- instruction->IsUnresolvedInstanceFieldGet() ||
- instruction->IsUnresolvedInstanceFieldSet() ||
- instruction->IsUnresolvedStaticFieldGet() ||
- instruction->IsUnresolvedStaticFieldSet();
+ instruction->IsTypeConversion();
}
bool HScheduler::IsSchedulable(const HBasicBlock* block) const {
diff --git a/compiler/optimizing/scheduler_arm.cc b/compiler/optimizing/scheduler_arm.cc
index ea15790..d6eb6e3 100644
--- a/compiler/optimizing/scheduler_arm.cc
+++ b/compiler/optimizing/scheduler_arm.cc
@@ -20,6 +20,7 @@
#include "code_generator_utils.h"
#include "common_arm.h"
#include "mirror/array-inl.h"
+#include "mirror/string.h"
namespace art {
namespace arm {
diff --git a/compiler/optimizing/scheduler_arm64.cc b/compiler/optimizing/scheduler_arm64.cc
index f54d3f3..510619f 100644
--- a/compiler/optimizing/scheduler_arm64.cc
+++ b/compiler/optimizing/scheduler_arm64.cc
@@ -18,6 +18,7 @@
#include "code_generator_utils.h"
#include "mirror/array-inl.h"
+#include "mirror/string.h"
namespace art {
namespace arm64 {
diff --git a/compiler/utils/arm/assembler_arm_vixl.cc b/compiler/utils/arm/assembler_arm_vixl.cc
index af3b447..9df1b74 100644
--- a/compiler/utils/arm/assembler_arm_vixl.cc
+++ b/compiler/utils/arm/assembler_arm_vixl.cc
@@ -82,6 +82,22 @@
}
}
+void ArmVIXLAssembler::GenerateMarkingRegisterCheck(vixl32::Register temp, int code) {
+ // The Marking Register is only used in the Baker read barrier configuration.
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+
+ vixl32::Label mr_is_ok;
+
+ // temp = self.tls32_.is.gc_marking
+ ___ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value()));
+ // Check that mr == self.tls32_.is.gc_marking.
+ ___ Cmp(mr, temp);
+ ___ B(eq, &mr_is_ok, /* far_target */ false);
+ ___ Bkpt(code);
+ ___ Bind(&mr_is_ok);
+}
+
void ArmVIXLAssembler::LoadImmediate(vixl32::Register rd, int32_t value) {
// TODO(VIXL): Implement this optimization in VIXL.
if (!ShifterOperandCanAlwaysHold(value) && ShifterOperandCanAlwaysHold(~value)) {
diff --git a/compiler/utils/arm/assembler_arm_vixl.h b/compiler/utils/arm/assembler_arm_vixl.h
index 66b22ea..9c11fd3 100644
--- a/compiler/utils/arm/assembler_arm_vixl.h
+++ b/compiler/utils/arm/assembler_arm_vixl.h
@@ -178,6 +178,7 @@
//
// Heap poisoning.
//
+
// Poison a heap reference contained in `reg`.
void PoisonHeapReference(vixl32::Register reg);
// Unpoison a heap reference contained in `reg`.
@@ -187,6 +188,15 @@
// Unpoison a heap reference contained in `reg` if heap poisoning is enabled.
void MaybeUnpoisonHeapReference(vixl32::Register reg);
+ // Emit code checking the status of the Marking Register, and aborting
+ // the program if MR does not match the value stored in the art::Thread
+ // object.
+ //
+ // Argument `temp` is used as a temporary register to generate code.
+ // Argument `code` is used to identify the different occurrences of
+ // MaybeGenerateMarkingRegisterCheck and is passed to the BKPT instruction.
+ void GenerateMarkingRegisterCheck(vixl32::Register temp, int code = 0);
+
void StoreToOffset(StoreOperandType type,
vixl32::Register reg,
vixl32::Register base,
diff --git a/compiler/utils/arm64/assembler_arm64.cc b/compiler/utils/arm64/assembler_arm64.cc
index 6ed0e9b..d8a48a5 100644
--- a/compiler/utils/arm64/assembler_arm64.cc
+++ b/compiler/utils/arm64/assembler_arm64.cc
@@ -158,6 +158,24 @@
}
}
+void Arm64Assembler::GenerateMarkingRegisterCheck(Register temp, int code) {
+ // The Marking Register is only used in the Baker read barrier configuration.
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+
+ vixl::aarch64::Register mr = reg_x(MR); // Marking Register.
+ vixl::aarch64::Register tr = reg_x(TR); // Thread Register.
+ vixl::aarch64::Label mr_is_ok;
+
+ // temp = self.tls32_.is.gc_marking
+ ___ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
+ // Check that mr == self.tls32_.is.gc_marking.
+ ___ Cmp(mr.W(), temp);
+ ___ B(eq, &mr_is_ok);
+ ___ Brk(code);
+ ___ Bind(&mr_is_ok);
+}
+
#undef ___
} // namespace arm64
diff --git a/compiler/utils/arm64/assembler_arm64.h b/compiler/utils/arm64/assembler_arm64.h
index 5b8a34e..6b28363 100644
--- a/compiler/utils/arm64/assembler_arm64.h
+++ b/compiler/utils/arm64/assembler_arm64.h
@@ -98,6 +98,15 @@
// Unpoison a heap reference contained in `reg` if heap poisoning is enabled.
void MaybeUnpoisonHeapReference(vixl::aarch64::Register reg);
+ // Emit code checking the status of the Marking Register, and aborting
+ // the program if MR does not match the value stored in the art::Thread
+ // object.
+ //
+ // Argument `temp` is used as a temporary register to generate code.
+ // Argument `code` is used to identify the different occurrences of
+ // MaybeGenerateMarkingRegisterCheck and is passed to the BRK instruction.
+ void GenerateMarkingRegisterCheck(vixl::aarch64::Register temp, int code = 0);
+
void Bind(Label* label ATTRIBUTE_UNUSED) OVERRIDE {
UNIMPLEMENTED(FATAL) << "Do not use Bind for ARM64";
}
diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.cc b/compiler/utils/arm64/jni_macro_assembler_arm64.cc
index bab84be..9732b76 100644
--- a/compiler/utils/arm64/jni_macro_assembler_arm64.cc
+++ b/compiler/utils/arm64/jni_macro_assembler_arm64.cc
@@ -662,7 +662,7 @@
___ Bind(Arm64JNIMacroLabel::Cast(label)->AsArm64());
}
-void Arm64JNIMacroAssembler::EmitExceptionPoll(Arm64Exception *exception) {
+void Arm64JNIMacroAssembler::EmitExceptionPoll(Arm64Exception* exception) {
UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
temps.Exclude(reg_x(exception->scratch_.AsXRegister()));
Register temp = temps.AcquireX();
diff --git a/compiler/utils/jni_macro_assembler.h b/compiler/utils/jni_macro_assembler.h
index 59a1a48..a8ca111 100644
--- a/compiler/utils/jni_macro_assembler.h
+++ b/compiler/utils/jni_macro_assembler.h
@@ -216,8 +216,15 @@
*/
virtual DebugFrameOpCodeWriterForAssembler& cfi() = 0;
+ void SetEmitRunTimeChecksInDebugMode(bool value) {
+ emit_run_time_checks_in_debug_mode_ = value;
+ }
+
protected:
- explicit JNIMacroAssembler() {}
+ JNIMacroAssembler() {}
+
+ // Should run-time checks be emitted in debug mode?
+ bool emit_run_time_checks_in_debug_mode_ = false;
};
// A "Label" class used with the JNIMacroAssembler
diff --git a/compiler/utils/x86_64/assembler_x86_64_test.cc b/compiler/utils/x86_64/assembler_x86_64_test.cc
index ec14e7a..9f2c44d 100644
--- a/compiler/utils/x86_64/assembler_x86_64_test.cc
+++ b/compiler/utils/x86_64/assembler_x86_64_test.cc
@@ -2012,7 +2012,7 @@
x86_64::X86_64ManagedRegister method_reg = ManagedFromCpu(x86_64::RDI);
size_t frame_size = 10 * kStackAlignment;
- assembler->BuildFrame(10 * kStackAlignment, method_reg, spill_regs, entry_spills);
+ assembler->BuildFrame(frame_size, method_reg, spill_regs, entry_spills);
// Construct assembly text counterpart.
std::ostringstream str;
@@ -2048,7 +2048,7 @@
ArrayRef<const ManagedRegister> spill_regs(raw_spill_regs);
size_t frame_size = 10 * kStackAlignment;
- assembler->RemoveFrame(10 * kStackAlignment, spill_regs);
+ assembler->RemoveFrame(frame_size, spill_regs);
// Construct assembly text counterpart.
std::ostringstream str;
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index e0ad649..e9ec5fa 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -355,6 +355,9 @@
UsageError("");
UsageError(" --debuggable: Produce code debuggable with Java debugger.");
UsageError("");
+ UsageError(" --avoid-storing-invocation: Avoid storing the invocation args in the key value");
+ UsageError(" store. Used to test determinism with different args.");
+ UsageError("");
UsageError(" --runtime-arg <argument>: used to specify various arguments for the runtime,");
UsageError(" such as initial heap size, maximum heap size, and verbose output.");
UsageError(" Use a separate --runtime-arg switch for each argument.");
@@ -611,6 +614,7 @@
dump_passes_(false),
dump_timing_(false),
dump_slow_timing_(kIsDebugBuild),
+ avoid_storing_invocation_(false),
swap_fd_(kInvalidFd),
app_image_fd_(kInvalidFd),
profile_file_fd_(kInvalidFd),
@@ -769,6 +773,11 @@
compiler_options_->boot_image_ = !image_filenames_.empty();
compiler_options_->app_image_ = app_image_fd_ != -1 || !app_image_file_name_.empty();
+ if (IsBootImage() && image_filenames_.size() == 1) {
+ const std::string& boot_image_filename = image_filenames_[0];
+ compiler_options_->core_image_ = CompilerDriver::IsCoreImageFilename(boot_image_filename);
+ }
+
if (IsAppImage() && IsBootImage()) {
Usage("Can't have both --image and (--app-image-fd or --app-image-file)");
}
@@ -1133,14 +1142,16 @@
void InsertCompileOptions(int argc, char** argv) {
std::ostringstream oss;
- for (int i = 0; i < argc; ++i) {
- if (i > 0) {
- oss << ' ';
+ if (!avoid_storing_invocation_) {
+ for (int i = 0; i < argc; ++i) {
+ if (i > 0) {
+ oss << ' ';
+ }
+ oss << argv[i];
}
- oss << argv[i];
+ key_value_store_->Put(OatHeader::kDex2OatCmdLineKey, oss.str());
+ oss.str(""); // Reset.
}
- key_value_store_->Put(OatHeader::kDex2OatCmdLineKey, oss.str());
- oss.str(""); // Reset.
oss << kRuntimeISA;
key_value_store_->Put(OatHeader::kDex2OatHostKey, oss.str());
key_value_store_->Put(
@@ -1271,6 +1282,8 @@
dump_passes_ = true;
} else if (option == "--dump-stats") {
dump_stats_ = true;
+ } else if (option == "--avoid-storing-invocation") {
+ avoid_storing_invocation_ = true;
} else if (option.starts_with("--swap-file=")) {
swap_file_name_ = option.substr(strlen("--swap-file=")).data();
} else if (option.starts_with("--swap-fd=")) {
@@ -1308,7 +1321,7 @@
} else if (option.starts_with("--class-loader-context=")) {
class_loader_context_ = ClassLoaderContext::Create(
option.substr(strlen("--class-loader-context=")).data());
- if (class_loader_context_== nullptr) {
+ if (class_loader_context_ == nullptr) {
Usage("Option --class-loader-context has an incorrect format: %s", option.data());
}
} else if (option.starts_with("--dirty-image-objects=")) {
@@ -1576,20 +1589,12 @@
}
// Open dex files for class path.
+
if (class_loader_context_ == nullptr) {
- // TODO(calin): Temporary workaround while we transition to use
- // --class-loader-context instead of --runtime-arg -cp
- if (runtime_->GetClassPathString().empty()) {
- class_loader_context_ = std::unique_ptr<ClassLoaderContext>(
- new ClassLoaderContext());
- } else {
- std::string spec = runtime_->GetClassPathString() == OatFile::kSpecialSharedLibrary
- ? OatFile::kSpecialSharedLibrary
- : "PCL[" + runtime_->GetClassPathString() + "]";
- class_loader_context_ = ClassLoaderContext::Create(spec);
- }
+ // If no context was specified use the default one (which is an empty PathClassLoader).
+ class_loader_context_ = std::unique_ptr<ClassLoaderContext>(ClassLoaderContext::Default());
}
- CHECK(class_loader_context_ != nullptr);
+
DCHECK_EQ(oat_writers_.size(), 1u);
// Note: Ideally we would reject context where the source dex files are also
@@ -1774,13 +1779,11 @@
bool ShouldCompileDexFilesIndividually() const {
// Compile individually if we are not building an image, not using any compilation, and are
// using multidex.
- // This means extract, verify, and quicken will use the individual compilation mode (to reduce
+ // This means extract, verify, and quicken, will use the individual compilation mode (to reduce
// RAM used by the compiler).
- // TODO: Still do it for app images to get testing coverage. Note that this will generate empty
- // app images.
return !IsImage() &&
dex_files_.size() > 1 &&
- !CompilerFilter::IsAnyCompilationEnabled(compiler_options_->GetCompilerFilter());
+ !CompilerFilter::IsAotCompilationEnabled(compiler_options_->GetCompilerFilter());
}
// Set up and create the compiler driver and then invoke it to compile all the dex files.
@@ -1856,6 +1859,16 @@
profile_compilation_info_.get()));
driver_->SetDexFilesForOatFile(dex_files_);
+ const bool compile_individually = ShouldCompileDexFilesIndividually();
+ if (compile_individually) {
+ // Set the compiler driver in the callbacks so that we can avoid re-verification. This not
+ // only helps performance but also prevents reverifying quickened bytecodes. Attempting
+ // verify quickened bytecode causes verification failures.
+ // Only set the compiler filter if we are doing separate compilation since there is a bit
+ // of overhead when checking if a class was previously verified.
+ callbacks_->SetDoesClassUnloading(true, driver_.get());
+ }
+
// Setup vdex for compilation.
if (!DoEagerUnquickeningOfVdex() && input_vdex_file_ != nullptr) {
callbacks_->SetVerifierDeps(
@@ -1872,7 +1885,7 @@
callbacks_->SetVerifierDeps(new verifier::VerifierDeps(dex_files_));
}
// Invoke the compilation.
- if (ShouldCompileDexFilesIndividually()) {
+ if (compile_individually) {
CompileDexFilesIndividually();
// Return a null classloader since we already freed released it.
return nullptr;
@@ -2899,6 +2912,7 @@
bool dump_passes_;
bool dump_timing_;
bool dump_slow_timing_;
+ bool avoid_storing_invocation_;
std::string swap_file_name_;
int swap_fd_;
size_t min_dex_files_for_swap_ = kDefaultMinDexFilesForSwap;
@@ -2965,6 +2979,8 @@
static dex2oat::ReturnCode CompileImage(Dex2Oat& dex2oat) {
dex2oat.LoadClassProfileDescriptors();
+ // Keep the class loader that was used for compilation live for the rest of the compilation
+ // process.
ScopedGlobalRef class_loader(dex2oat.Compile());
if (!dex2oat.WriteOutputFiles()) {
@@ -3013,6 +3029,8 @@
}
static dex2oat::ReturnCode CompileApp(Dex2Oat& dex2oat) {
+ // Keep the class loader that was used for compilation live for the rest of the compilation
+ // process.
ScopedGlobalRef class_loader(dex2oat.Compile());
if (!dex2oat.WriteOutputFiles()) {
diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc
index 12bceb3..6a9d979 100644
--- a/dex2oat/dex2oat_test.cc
+++ b/dex2oat/dex2oat_test.cc
@@ -41,6 +41,7 @@
namespace art {
static constexpr size_t kMaxMethodIds = 65535;
+static constexpr bool kDebugArgs = false;
using android::base::StringPrintf;
@@ -55,7 +56,7 @@
}
protected:
- int GenerateOdexForTestWithStatus(const std::string& dex_location,
+ int GenerateOdexForTestWithStatus(const std::vector<std::string>& dex_locations,
const std::string& odex_location,
CompilerFilter::Filter filter,
std::string* error_msg,
@@ -63,7 +64,10 @@
bool use_fd = false) {
std::unique_ptr<File> oat_file;
std::vector<std::string> args;
- args.push_back("--dex-file=" + dex_location);
+ // Add dex file args.
+ for (const std::string& dex_location : dex_locations) {
+ args.push_back("--dex-file=" + dex_location);
+ }
if (use_fd) {
oat_file.reset(OS::CreateEmptyFile(odex_location.c_str()));
CHECK(oat_file != nullptr) << odex_location;
@@ -93,7 +97,7 @@
bool use_fd = false,
std::function<void(const OatFile&)> check_oat = [](const OatFile&) {}) {
std::string error_msg;
- int status = GenerateOdexForTestWithStatus(dex_location,
+ int status = GenerateOdexForTestWithStatus({dex_location},
odex_location,
filter,
&error_msg,
@@ -187,6 +191,14 @@
CHECK(android_root != nullptr);
argv.push_back("--android-root=" + std::string(android_root));
+ if (kDebugArgs) {
+ std::string all_args;
+ for (const std::string& arg : argv) {
+ all_args += arg + " ";
+ }
+ LOG(ERROR) << all_args;
+ }
+
int link[2];
if (pipe(link) == -1) {
@@ -951,7 +963,7 @@
Copy(GetTestDexFileName(), dex_location);
std::string error_msg;
- return GenerateOdexForTestWithStatus(dex_location,
+ return GenerateOdexForTestWithStatus({dex_location},
odex_location,
CompilerFilter::kSpeed,
&error_msg,
@@ -1107,4 +1119,72 @@
RunTest(context.c_str(), expected_classpath_key.c_str(), true);
}
+class Dex2oatDeterminism : public Dex2oatTest {};
+
+TEST_F(Dex2oatDeterminism, UnloadCompile) {
+ if (!kUseReadBarrier &&
+ gc::kCollectorTypeDefault != gc::kCollectorTypeCMS &&
+ gc::kCollectorTypeDefault != gc::kCollectorTypeMS) {
+ LOG(INFO) << "Test requires determinism support.";
+ return;
+ }
+ Runtime* const runtime = Runtime::Current();
+ std::string out_dir = GetScratchDir();
+ const std::string base_oat_name = out_dir + "/base.oat";
+ const std::string base_vdex_name = out_dir + "/base.vdex";
+ const std::string unload_oat_name = out_dir + "/unload.oat";
+ const std::string unload_vdex_name = out_dir + "/unload.vdex";
+ const std::string no_unload_oat_name = out_dir + "/nounload.oat";
+ const std::string no_unload_vdex_name = out_dir + "/nounload.vdex";
+ const std::string app_image_name = out_dir + "/unload.art";
+ std::string error_msg;
+ const std::vector<gc::space::ImageSpace*>& spaces = runtime->GetHeap()->GetBootImageSpaces();
+ ASSERT_GT(spaces.size(), 0u);
+ const std::string image_location = spaces[0]->GetImageLocation();
+ // Without passing in an app image, it will unload in between compilations.
+ const int res = GenerateOdexForTestWithStatus(
+ GetLibCoreDexFileNames(),
+ base_oat_name,
+ CompilerFilter::Filter::kQuicken,
+ &error_msg,
+ {"--force-determinism", "--avoid-storing-invocation"});
+ EXPECT_EQ(res, 0);
+ Copy(base_oat_name, unload_oat_name);
+ Copy(base_vdex_name, unload_vdex_name);
+ std::unique_ptr<File> unload_oat(OS::OpenFileForReading(unload_oat_name.c_str()));
+ std::unique_ptr<File> unload_vdex(OS::OpenFileForReading(unload_vdex_name.c_str()));
+ ASSERT_TRUE(unload_oat != nullptr);
+ ASSERT_TRUE(unload_vdex != nullptr);
+ EXPECT_GT(unload_oat->GetLength(), 0u);
+ EXPECT_GT(unload_vdex->GetLength(), 0u);
+ // Regenerate with an app image to disable the dex2oat unloading and verify that the output is
+ // the same.
+ const int res2 = GenerateOdexForTestWithStatus(
+ GetLibCoreDexFileNames(),
+ base_oat_name,
+ CompilerFilter::Filter::kQuicken,
+ &error_msg,
+ {"--force-determinism", "--avoid-storing-invocation", "--app-image-file=" + app_image_name});
+ EXPECT_EQ(res2, 0);
+ Copy(base_oat_name, no_unload_oat_name);
+ Copy(base_vdex_name, no_unload_vdex_name);
+ std::unique_ptr<File> no_unload_oat(OS::OpenFileForReading(no_unload_oat_name.c_str()));
+ std::unique_ptr<File> no_unload_vdex(OS::OpenFileForReading(no_unload_vdex_name.c_str()));
+ ASSERT_TRUE(no_unload_oat != nullptr);
+ ASSERT_TRUE(no_unload_vdex != nullptr);
+ EXPECT_GT(no_unload_oat->GetLength(), 0u);
+ EXPECT_GT(no_unload_vdex->GetLength(), 0u);
+ // Verify that both of the files are the same (odex and vdex).
+ EXPECT_EQ(unload_oat->GetLength(), no_unload_oat->GetLength());
+ EXPECT_EQ(unload_vdex->GetLength(), no_unload_vdex->GetLength());
+ EXPECT_EQ(unload_oat->Compare(no_unload_oat.get()), 0)
+ << unload_oat_name << " " << no_unload_oat_name;
+ EXPECT_EQ(unload_vdex->Compare(no_unload_vdex.get()), 0)
+ << unload_vdex_name << " " << no_unload_vdex_name;
+ // App image file.
+ std::unique_ptr<File> app_image_file(OS::OpenFileForReading(app_image_name.c_str()));
+ ASSERT_TRUE(app_image_file != nullptr);
+ EXPECT_GT(app_image_file->GetLength(), 0u);
+}
+
} // namespace art
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index 99168c9..f8b1f53 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -89,6 +89,8 @@
"kSaveRefsOnlyMethod",
"kSaveRefsAndArgsMethod",
"kSaveEverythingMethod",
+ "kSaveEverythingMethodForClinit",
+ "kSaveEverythingMethodForSuspendCheck",
};
const char* image_roots_descriptions_[] = {
diff --git a/runtime/aot_class_linker.cc b/runtime/aot_class_linker.cc
index b1bc3f8..9396565 100644
--- a/runtime/aot_class_linker.cc
+++ b/runtime/aot_class_linker.cc
@@ -16,10 +16,12 @@
#include "aot_class_linker.h"
+#include "class_reference.h"
+#include "compiler_callbacks.h"
#include "handle_scope-inl.h"
-#include "mirror/class.h"
-#include "mirror/object-inl.h"
+#include "mirror/class-inl.h"
#include "runtime.h"
+#include "verifier/verifier_enums.h"
namespace art {
@@ -66,4 +68,18 @@
}
return success;
}
+
+verifier::FailureKind AotClassLinker::PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg) {
+ Runtime* const runtime = Runtime::Current();
+ CompilerCallbacks* callbacks = runtime->GetCompilerCallbacks();
+ if (callbacks->CanAssumeVerified(ClassReference(&klass->GetDexFile(),
+ klass->GetDexClassDefIndex()))) {
+ return verifier::FailureKind::kNoFailure;
+ }
+ return ClassLinker::PerformClassVerification(self, klass, log_level, error_msg);
+}
+
} // namespace art
diff --git a/runtime/aot_class_linker.h b/runtime/aot_class_linker.h
index 11bea86..927b533 100644
--- a/runtime/aot_class_linker.h
+++ b/runtime/aot_class_linker.h
@@ -27,6 +27,16 @@
explicit AotClassLinker(InternTable *intern_table);
~AotClassLinker();
+ protected:
+ // Overridden version of PerformClassVerification allows skipping verification if the class was
+ // previously verified but unloaded.
+ verifier::FailureKind PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg)
+ OVERRIDE
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
bool InitializeClass(Thread *self,
Handle<mirror::Class> klass,
bool can_run_clinit,
diff --git a/runtime/arch/arch_test.cc b/runtime/arch/arch_test.cc
index bfac8c4..1ba4070 100644
--- a/runtime/arch/arch_test.cc
+++ b/runtime/arch/arch_test.cc
@@ -88,6 +88,11 @@
#undef FRAME_SIZE_SAVE_REFS_ONLY
static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS;
#undef FRAME_SIZE_SAVE_REFS_AND_ARGS
+static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT
+static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck =
+ FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK
static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING;
#undef FRAME_SIZE_SAVE_EVERYTHING
#undef BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET
@@ -109,6 +114,11 @@
#undef FRAME_SIZE_SAVE_REFS_ONLY
static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS;
#undef FRAME_SIZE_SAVE_REFS_AND_ARGS
+static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT
+static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck =
+ FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK
static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING;
#undef FRAME_SIZE_SAVE_EVERYTHING
#undef BAKER_MARK_INTROSPECTION_ARRAY_SWITCH_OFFSET
@@ -126,6 +136,11 @@
#undef FRAME_SIZE_SAVE_REFS_ONLY
static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS;
#undef FRAME_SIZE_SAVE_REFS_AND_ARGS
+static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT
+static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck =
+ FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK
static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING;
#undef FRAME_SIZE_SAVE_EVERYTHING
#undef BAKER_MARK_INTROSPECTION_REGISTER_COUNT
@@ -142,6 +157,11 @@
#undef FRAME_SIZE_SAVE_REFS_ONLY
static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS;
#undef FRAME_SIZE_SAVE_REFS_AND_ARGS
+static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT
+static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck =
+ FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK
static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING;
#undef FRAME_SIZE_SAVE_EVERYTHING
#undef BAKER_MARK_INTROSPECTION_REGISTER_COUNT
@@ -158,6 +178,11 @@
#undef FRAME_SIZE_SAVE_REFS_ONLY
static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS;
#undef FRAME_SIZE_SAVE_REFS_AND_ARGS
+static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT
+static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck =
+ FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK
static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING;
#undef FRAME_SIZE_SAVE_EVERYTHING
} // namespace x86
@@ -170,25 +195,36 @@
#undef FRAME_SIZE_SAVE_REFS_ONLY
static constexpr size_t kFrameSizeSaveRefsAndArgs = FRAME_SIZE_SAVE_REFS_AND_ARGS;
#undef FRAME_SIZE_SAVE_REFS_AND_ARGS
+static constexpr size_t kFrameSizeSaveEverythingForClinit = FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT
+static constexpr size_t kFrameSizeSaveEverythingForSuspendCheck =
+ FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK;
+#undef FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK
static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING;
#undef FRAME_SIZE_SAVE_EVERYTHING
} // namespace x86_64
// Check architecture specific constants are sound.
-#define TEST_ARCH(Arch, arch) \
- TEST_F(ArchTest, Arch) { \
- CheckFrameSize(InstructionSet::k##Arch, \
- CalleeSaveType::kSaveAllCalleeSaves, \
- arch::kFrameSizeSaveAllCalleeSaves); \
- CheckFrameSize(InstructionSet::k##Arch, \
- CalleeSaveType::kSaveRefsOnly, \
- arch::kFrameSizeSaveRefsOnly); \
- CheckFrameSize(InstructionSet::k##Arch, \
- CalleeSaveType::kSaveRefsAndArgs, \
- arch::kFrameSizeSaveRefsAndArgs); \
- CheckFrameSize(InstructionSet::k##Arch, \
- CalleeSaveType::kSaveEverything, \
- arch::kFrameSizeSaveEverything); \
+#define TEST_ARCH(Arch, arch) \
+ TEST_F(ArchTest, Arch) { \
+ CheckFrameSize(InstructionSet::k##Arch, \
+ CalleeSaveType::kSaveAllCalleeSaves, \
+ arch::kFrameSizeSaveAllCalleeSaves); \
+ CheckFrameSize(InstructionSet::k##Arch, \
+ CalleeSaveType::kSaveRefsOnly, \
+ arch::kFrameSizeSaveRefsOnly); \
+ CheckFrameSize(InstructionSet::k##Arch, \
+ CalleeSaveType::kSaveRefsAndArgs, \
+ arch::kFrameSizeSaveRefsAndArgs); \
+ CheckFrameSize(InstructionSet::k##Arch, \
+ CalleeSaveType::kSaveEverything, \
+ arch::kFrameSizeSaveEverything); \
+ CheckFrameSize(InstructionSet::k##Arch, \
+ CalleeSaveType::kSaveEverythingForClinit, \
+ arch::kFrameSizeSaveEverythingForClinit); \
+ CheckFrameSize(InstructionSet::k##Arch, \
+ CalleeSaveType::kSaveEverythingForSuspendCheck, \
+ arch::kFrameSizeSaveEverythingForSuspendCheck); \
}
TEST_ARCH(Arm, arm)
TEST_ARCH(Arm64, arm64)
diff --git a/runtime/arch/arm/asm_support_arm.h b/runtime/arch/arm/asm_support_arm.h
index 8f2fd6e..3d85872 100644
--- a/runtime/arch/arm/asm_support_arm.h
+++ b/runtime/arch/arm/asm_support_arm.h
@@ -23,6 +23,8 @@
#define FRAME_SIZE_SAVE_REFS_ONLY 32
#define FRAME_SIZE_SAVE_REFS_AND_ARGS 112
#define FRAME_SIZE_SAVE_EVERYTHING 192
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING
// The offset from the art_quick_read_barrier_mark_introspection (used for field
// loads with 32-bit LDR) to the entrypoint for field loads with 16-bit LDR,
diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S
index 375768e..cf2bfee 100644
--- a/runtime/arch/arm/quick_entrypoints_arm.S
+++ b/runtime/arch/arm/quick_entrypoints_arm.S
@@ -182,7 +182,7 @@
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
* when core registers are already saved.
*/
-.macro SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED rTemp
+.macro SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED rTemp, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET
@ 14 words of callee saves and args already saved.
vpush {d0-d15} @ 32 words, 2 for each of the 16 saved doubles.
.cfi_adjust_cfa_offset 128
@@ -190,7 +190,7 @@
.cfi_adjust_cfa_offset 8
RUNTIME_CURRENT1 \rTemp @ Load Runtime::Current into rTemp.
@ Load kSaveEverything Method* into rTemp.
- ldr \rTemp, [\rTemp, #RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET]
+ ldr \rTemp, [\rTemp, #\runtime_method_offset]
str \rTemp, [sp, #0] @ Place Method* at bottom of stack.
str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame.
@@ -204,7 +204,7 @@
* Macro that sets up the callee save frame to conform with
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
*/
-.macro SETUP_SAVE_EVERYTHING_FRAME rTemp
+.macro SETUP_SAVE_EVERYTHING_FRAME rTemp, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET
push {r0-r12, lr} @ 14 words of callee saves and args.
.cfi_adjust_cfa_offset 56
.cfi_rel_offset r0, 0
@@ -221,7 +221,7 @@
.cfi_rel_offset r11, 44
.cfi_rel_offset ip, 48
.cfi_rel_offset lr, 52
- SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED \rTemp
+ SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED \rTemp, \runtime_method_offset
.endm
.macro RESTORE_SAVE_EVERYTHING_FRAME
@@ -1004,10 +1004,10 @@
.endm
// Macro for string and type resolution and initialization.
-.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint
+.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET
.extern \entrypoint
ENTRY \name
- SETUP_SAVE_EVERYTHING_FRAME r1 @ save everything in case of GC
+ SETUP_SAVE_EVERYTHING_FRAME r1, \runtime_method_offset @ save everything in case of GC
mov r1, r9 @ pass Thread::Current
bl \entrypoint @ (uint32_t index, Thread*)
cbz r0, 1f @ If result is null, deliver the OOME.
@@ -1021,8 +1021,12 @@
END \name
.endm
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode
+.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint
+ ONE_ARG_SAVE_EVERYTHING_DOWNCALL \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET
+.endm
+
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode
@@ -1537,7 +1541,7 @@
1:
mov rSUSPEND, #SUSPEND_CHECK_INTERVAL @ reset rSUSPEND to SUSPEND_CHECK_INTERVAL
#endif
- SETUP_SAVE_EVERYTHING_FRAME r0 @ save everything for GC stack crawl
+ SETUP_SAVE_EVERYTHING_FRAME r0, RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET @ save everything for GC stack crawl
mov r0, rSELF
bl artTestSuspendFromCode @ (Thread*)
RESTORE_SAVE_EVERYTHING_FRAME
diff --git a/runtime/arch/arm/quick_method_frame_info_arm.h b/runtime/arch/arm/quick_method_frame_info_arm.h
index 39061f0..5c5b81b 100644
--- a/runtime/arch/arm/quick_method_frame_info_arm.h
+++ b/runtime/arch/arm/quick_method_frame_info_arm.h
@@ -56,6 +56,7 @@
kArmCalleeSaveFpArgSpills | kArmCalleeSaveFpAllSpills;
constexpr uint32_t ArmCalleeSaveCoreSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kArmCalleeSaveAlwaysSpills | kArmCalleeSaveRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kArmCalleeSaveArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kArmCalleeSaveAllSpills : 0) |
@@ -63,6 +64,7 @@
}
constexpr uint32_t ArmCalleeSaveFpSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kArmCalleeSaveFpAlwaysSpills | kArmCalleeSaveFpRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kArmCalleeSaveFpArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kArmCalleeSaveFpAllSpills : 0) |
@@ -70,29 +72,34 @@
}
constexpr uint32_t ArmCalleeSaveFrameSize(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return RoundUp((POPCOUNT(ArmCalleeSaveCoreSpills(type)) /* gprs */ +
POPCOUNT(ArmCalleeSaveFpSpills(type)) /* fprs */ +
1 /* Method* */) * static_cast<size_t>(kArmPointerSize), kStackAlignment);
}
constexpr QuickMethodFrameInfo ArmCalleeSaveMethodFrameInfo(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return QuickMethodFrameInfo(ArmCalleeSaveFrameSize(type),
ArmCalleeSaveCoreSpills(type),
ArmCalleeSaveFpSpills(type));
}
constexpr size_t ArmCalleeSaveFpr1Offset(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return ArmCalleeSaveFrameSize(type) -
(POPCOUNT(ArmCalleeSaveCoreSpills(type)) +
POPCOUNT(ArmCalleeSaveFpSpills(type))) * static_cast<size_t>(kArmPointerSize);
}
constexpr size_t ArmCalleeSaveGpr1Offset(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return ArmCalleeSaveFrameSize(type) -
POPCOUNT(ArmCalleeSaveCoreSpills(type)) * static_cast<size_t>(kArmPointerSize);
}
constexpr size_t ArmCalleeSaveLrOffset(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return ArmCalleeSaveFrameSize(type) -
POPCOUNT(ArmCalleeSaveCoreSpills(type) & (-(1 << LR))) * static_cast<size_t>(kArmPointerSize);
}
diff --git a/runtime/arch/arm64/asm_support_arm64.h b/runtime/arch/arm64/asm_support_arm64.h
index 6b77200..57376d0 100644
--- a/runtime/arch/arm64/asm_support_arm64.h
+++ b/runtime/arch/arm64/asm_support_arm64.h
@@ -23,6 +23,8 @@
#define FRAME_SIZE_SAVE_REFS_ONLY 96
#define FRAME_SIZE_SAVE_REFS_AND_ARGS 224
#define FRAME_SIZE_SAVE_EVERYTHING 512
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING
// The offset from art_quick_read_barrier_mark_introspection to the array switch cases,
// i.e. art_quick_read_barrier_mark_introspection_arrays.
diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S
index d15f5b8..3d8ca40 100644
--- a/runtime/arch/arm64/quick_entrypoints_arm64.S
+++ b/runtime/arch/arm64/quick_entrypoints_arm64.S
@@ -287,7 +287,7 @@
* when the SP has already been decremented by FRAME_SIZE_SAVE_EVERYTHING
* and saving registers x29 and LR is handled elsewhere.
*/
-.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR
+.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET
// Ugly compile-time check, but we only have the preprocessor.
#if (FRAME_SIZE_SAVE_EVERYTHING != 512)
#error "FRAME_SIZE_SAVE_EVERYTHING(ARM64) size not as expected."
@@ -337,7 +337,7 @@
ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_;
// ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveEverything];
- ldr xIP0, [xIP0, RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET]
+ ldr xIP0, [xIP0, \runtime_method_offset]
// Store ArtMethod* Runtime::callee_save_methods_[kSaveEverything].
str xIP0, [sp]
@@ -350,10 +350,10 @@
* Macro that sets up the callee save frame to conform with
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
*/
-.macro SETUP_SAVE_EVERYTHING_FRAME
+.macro SETUP_SAVE_EVERYTHING_FRAME runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET
INCREASE_FRAME 512
SAVE_TWO_REGS x29, xLR, 496
- SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR
+ SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR \runtime_method_offset
.endm
.macro RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0
@@ -398,7 +398,7 @@
.endm
.macro RESTORE_SAVE_EVERYTHING_FRAME
- RESTORE_REG x0, 264
+ RESTORE_REG x0, 264
RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0
.endm
@@ -1593,10 +1593,10 @@
.endm
// Macro for string and type resolution and initialization.
-.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint
+.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET
.extern \entrypoint
ENTRY \name
- SETUP_SAVE_EVERYTHING_FRAME // save everything for stack crawl
+ SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for stack crawl
mov x1, xSELF // pass Thread::Current
bl \entrypoint // (int32_t index, Thread* self)
cbz w0, 1f // If result is null, deliver the OOME.
@@ -1611,6 +1611,10 @@
END \name
.endm
+.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint
+ ONE_ARG_SAVE_EVERYTHING_DOWNCALL \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET
+.endm
+
.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER
cbz w0, 1f // result zero branch over
ret // return
@@ -1629,9 +1633,8 @@
* initializer and deliver the exception on error. On success the static storage base is
* returned.
*/
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
-
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode
@@ -2008,7 +2011,7 @@
*/
.extern artTestSuspendFromCode
ENTRY art_quick_test_suspend
- SETUP_SAVE_EVERYTHING_FRAME // save callee saves for stack crawl
+ SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save callee saves for stack crawl
mov x0, xSELF
bl artTestSuspendFromCode // (Thread*)
RESTORE_SAVE_EVERYTHING_FRAME
diff --git a/runtime/arch/arm64/quick_method_frame_info_arm64.h b/runtime/arch/arm64/quick_method_frame_info_arm64.h
index c231d4d..3e6f6c6 100644
--- a/runtime/arch/arm64/quick_method_frame_info_arm64.h
+++ b/runtime/arch/arm64/quick_method_frame_info_arm64.h
@@ -80,6 +80,7 @@
(1 << art::arm64::D30) | (1 << art::arm64::D31);
constexpr uint32_t Arm64CalleeSaveCoreSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kArm64CalleeSaveAlwaysSpills | kArm64CalleeSaveRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kArm64CalleeSaveArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kArm64CalleeSaveAllSpills : 0) |
@@ -87,6 +88,7 @@
}
constexpr uint32_t Arm64CalleeSaveFpSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kArm64CalleeSaveFpAlwaysSpills | kArm64CalleeSaveFpRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kArm64CalleeSaveFpArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kArm64CalleeSaveFpAllSpills : 0) |
@@ -94,29 +96,34 @@
}
constexpr uint32_t Arm64CalleeSaveFrameSize(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return RoundUp((POPCOUNT(Arm64CalleeSaveCoreSpills(type)) /* gprs */ +
POPCOUNT(Arm64CalleeSaveFpSpills(type)) /* fprs */ +
1 /* Method* */) * static_cast<size_t>(kArm64PointerSize), kStackAlignment);
}
constexpr QuickMethodFrameInfo Arm64CalleeSaveMethodFrameInfo(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return QuickMethodFrameInfo(Arm64CalleeSaveFrameSize(type),
Arm64CalleeSaveCoreSpills(type),
Arm64CalleeSaveFpSpills(type));
}
constexpr size_t Arm64CalleeSaveFpr1Offset(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return Arm64CalleeSaveFrameSize(type) -
(POPCOUNT(Arm64CalleeSaveCoreSpills(type)) +
POPCOUNT(Arm64CalleeSaveFpSpills(type))) * static_cast<size_t>(kArm64PointerSize);
}
constexpr size_t Arm64CalleeSaveGpr1Offset(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return Arm64CalleeSaveFrameSize(type) -
POPCOUNT(Arm64CalleeSaveCoreSpills(type)) * static_cast<size_t>(kArm64PointerSize);
}
constexpr size_t Arm64CalleeSaveLrOffset(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return Arm64CalleeSaveFrameSize(type) -
POPCOUNT(Arm64CalleeSaveCoreSpills(type) & (-(1 << LR))) *
static_cast<size_t>(kArm64PointerSize);
diff --git a/runtime/arch/mips/asm_support_mips.h b/runtime/arch/mips/asm_support_mips.h
index 9d8572f..2edd63f 100644
--- a/runtime/arch/mips/asm_support_mips.h
+++ b/runtime/arch/mips/asm_support_mips.h
@@ -23,6 +23,8 @@
#define FRAME_SIZE_SAVE_REFS_ONLY 48
#define FRAME_SIZE_SAVE_REFS_AND_ARGS 112
#define FRAME_SIZE_SAVE_EVERYTHING 256
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING
// &art_quick_read_barrier_mark_introspection is the first of many entry points:
// 21 entry points for long field offsets, large array indices and variable array indices
diff --git a/runtime/arch/mips/quick_method_frame_info_mips.h b/runtime/arch/mips/quick_method_frame_info_mips.h
index 01879a5..45a21ab 100644
--- a/runtime/arch/mips/quick_method_frame_info_mips.h
+++ b/runtime/arch/mips/quick_method_frame_info_mips.h
@@ -65,6 +65,7 @@
(1 << art::mips::F28) | (1 << art::mips::F29) | (1 << art::mips::F30) | (1u << art::mips::F31);
constexpr uint32_t MipsCalleeSaveCoreSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kMipsCalleeSaveAlwaysSpills | kMipsCalleeSaveRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kMipsCalleeSaveArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kMipsCalleeSaveAllSpills : 0) |
@@ -72,6 +73,7 @@
}
constexpr uint32_t MipsCalleeSaveFPSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kMipsCalleeSaveFpAlwaysSpills | kMipsCalleeSaveFpRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kMipsCalleeSaveFpArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kMipsCalleeSaveAllFPSpills : 0) |
@@ -79,12 +81,14 @@
}
constexpr uint32_t MipsCalleeSaveFrameSize(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return RoundUp((POPCOUNT(MipsCalleeSaveCoreSpills(type)) /* gprs */ +
POPCOUNT(MipsCalleeSaveFPSpills(type)) /* fprs */ +
1 /* Method* */) * static_cast<size_t>(kMipsPointerSize), kStackAlignment);
}
constexpr QuickMethodFrameInfo MipsCalleeSaveMethodFrameInfo(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return QuickMethodFrameInfo(MipsCalleeSaveFrameSize(type),
MipsCalleeSaveCoreSpills(type),
MipsCalleeSaveFPSpills(type));
diff --git a/runtime/arch/mips64/asm_support_mips64.h b/runtime/arch/mips64/asm_support_mips64.h
index 7185da5..a8e907e 100644
--- a/runtime/arch/mips64/asm_support_mips64.h
+++ b/runtime/arch/mips64/asm_support_mips64.h
@@ -27,6 +27,8 @@
#define FRAME_SIZE_SAVE_REFS_AND_ARGS 208
// $f0-$f31, $at, $v0-$v1, $a0-$a7, $t0-$t3, $s0-$s7, $t8-$t9, $gp, $s8, $ra + padding + method*
#define FRAME_SIZE_SAVE_EVERYTHING 496
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING
// &art_quick_read_barrier_mark_introspection is the first of many entry points:
// 20 entry points for long field offsets, large array indices and variable array indices
diff --git a/runtime/arch/mips64/quick_method_frame_info_mips64.h b/runtime/arch/mips64/quick_method_frame_info_mips64.h
index a55ab0e..520f631 100644
--- a/runtime/arch/mips64/quick_method_frame_info_mips64.h
+++ b/runtime/arch/mips64/quick_method_frame_info_mips64.h
@@ -72,6 +72,7 @@
(1 << art::mips64::F30) | (1 << art::mips64::F31);
constexpr uint32_t Mips64CalleeSaveCoreSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kMips64CalleeSaveAlwaysSpills | kMips64CalleeSaveRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kMips64CalleeSaveArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kMips64CalleeSaveAllSpills : 0) |
@@ -79,6 +80,7 @@
}
constexpr uint32_t Mips64CalleeSaveFpSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kMips64CalleeSaveFpRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kMips64CalleeSaveFpArgSpills : 0) |
(type == CalleeSaveType::kSaveAllCalleeSaves ? kMips64CalleeSaveFpAllSpills : 0) |
@@ -86,12 +88,14 @@
}
constexpr uint32_t Mips64CalleeSaveFrameSize(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return RoundUp((POPCOUNT(Mips64CalleeSaveCoreSpills(type)) /* gprs */ +
POPCOUNT(Mips64CalleeSaveFpSpills(type)) /* fprs */ +
+ 1 /* Method* */) * static_cast<size_t>(kMips64PointerSize), kStackAlignment);
}
constexpr QuickMethodFrameInfo Mips64CalleeSaveMethodFrameInfo(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return QuickMethodFrameInfo(Mips64CalleeSaveFrameSize(type),
Mips64CalleeSaveCoreSpills(type),
Mips64CalleeSaveFpSpills(type));
diff --git a/runtime/arch/x86/asm_support_x86.h b/runtime/arch/x86/asm_support_x86.h
index 2bba08d..737d736 100644
--- a/runtime/arch/x86/asm_support_x86.h
+++ b/runtime/arch/x86/asm_support_x86.h
@@ -23,5 +23,7 @@
#define FRAME_SIZE_SAVE_REFS_ONLY 32
#define FRAME_SIZE_SAVE_REFS_AND_ARGS (32 + 32)
#define FRAME_SIZE_SAVE_EVERYTHING (48 + 64)
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING
#endif // ART_RUNTIME_ARCH_X86_ASM_SUPPORT_X86_H_
diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S
index 48d2de9..4e5e93a 100644
--- a/runtime/arch/x86/quick_entrypoints_x86.S
+++ b/runtime/arch/x86/quick_entrypoints_x86.S
@@ -226,7 +226,7 @@
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
* when EDI and ESI are already saved.
*/
-MACRO2(SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED, got_reg, temp_reg)
+MACRO3(SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED, got_reg, temp_reg, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
// Save core registers from highest to lowest to agree with core spills bitmap.
// EDI and ESI, or at least placeholders for them, are already on the stack.
PUSH ebp
@@ -252,7 +252,7 @@
movl SYMBOL(_ZN3art7Runtime9instance_E)@GOT(REG_VAR(got_reg)), REG_VAR(temp_reg)
movl (REG_VAR(temp_reg)), REG_VAR(temp_reg)
// Push save everything callee-save method.
- pushl RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET(REG_VAR(temp_reg))
+ pushl \runtime_method_offset(REG_VAR(temp_reg))
CFI_ADJUST_CFA_OFFSET(4)
// Store esp as the stop quick frame.
movl %esp, %fs:THREAD_TOP_QUICK_FRAME_OFFSET
@@ -269,20 +269,20 @@
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
* when EDI is already saved.
*/
-MACRO2(SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED, got_reg, temp_reg)
+MACRO3(SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED, got_reg, temp_reg, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
// Save core registers from highest to lowest to agree with core spills bitmap.
// EDI, or at least a placeholder for it, is already on the stack.
PUSH esi
- SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg)
+ SETUP_SAVE_EVERYTHING_FRAME_EDI_ESI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg), \runtime_method_offset
END_MACRO
/*
* Macro that sets up the callee save frame to conform with
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
*/
-MACRO2(SETUP_SAVE_EVERYTHING_FRAME, got_reg, temp_reg)
+MACRO3(SETUP_SAVE_EVERYTHING_FRAME, got_reg, temp_reg, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
PUSH edi
- SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg)
+ SETUP_SAVE_EVERYTHING_FRAME_EDI_SAVED RAW_VAR(got_reg), RAW_VAR(temp_reg), \runtime_method_offset
END_MACRO
MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_FRPS)
@@ -923,9 +923,9 @@
END_MACRO
// Macro for string and type resolution and initialization.
-MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name)
+MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
DEFINE_FUNCTION VAR(c_name)
- SETUP_SAVE_EVERYTHING_FRAME ebx, ebx // save ref containing registers for GC
+ SETUP_SAVE_EVERYTHING_FRAME ebx, ebx, \runtime_method_offset // save ref containing registers for GC
// Outgoing argument set up
subl MACRO_LITERAL(8), %esp // push padding
CFI_ADJUST_CFA_OFFSET(8)
@@ -947,6 +947,10 @@
END_FUNCTION VAR(c_name)
END_MACRO
+MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name)
+ ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET
+END_MACRO
+
MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER)
testl %eax, %eax // eax == 0 ?
jz 1f // if eax == 0 goto 1
@@ -1270,8 +1274,8 @@
GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_32
GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_64
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode
@@ -1598,7 +1602,7 @@
END_FUNCTION art_quick_memcpy
DEFINE_FUNCTION art_quick_test_suspend
- SETUP_SAVE_EVERYTHING_FRAME ebx, ebx // save everything for GC
+ SETUP_SAVE_EVERYTHING_FRAME ebx, ebx, RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save everything for GC
// Outgoing argument set up
subl MACRO_LITERAL(12), %esp // push padding
CFI_ADJUST_CFA_OFFSET(12)
diff --git a/runtime/arch/x86/quick_method_frame_info_x86.h b/runtime/arch/x86/quick_method_frame_info_x86.h
index 8342c9f..9a66333 100644
--- a/runtime/arch/x86/quick_method_frame_info_x86.h
+++ b/runtime/arch/x86/quick_method_frame_info_x86.h
@@ -57,23 +57,27 @@
(1 << art::x86::XMM6) | (1 << art::x86::XMM7);
constexpr uint32_t X86CalleeSaveCoreSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kX86CalleeSaveAlwaysSpills | kX86CalleeSaveRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveArgSpills : 0) |
(type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveEverythingSpills : 0);
}
constexpr uint32_t X86CalleeSaveFpSpills(CalleeSaveType type) {
- return (type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveFpArgSpills : 0) |
- (type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveFpEverythingSpills : 0);
+ type = GetCanonicalCalleeSaveType(type);
+ return (type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveFpArgSpills : 0) |
+ (type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveFpEverythingSpills : 0);
}
constexpr uint32_t X86CalleeSaveFrameSize(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return RoundUp((POPCOUNT(X86CalleeSaveCoreSpills(type)) /* gprs */ +
2 * POPCOUNT(X86CalleeSaveFpSpills(type)) /* fprs */ +
1 /* Method* */) * static_cast<size_t>(kX86PointerSize), kStackAlignment);
}
constexpr QuickMethodFrameInfo X86CalleeSaveMethodFrameInfo(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return QuickMethodFrameInfo(X86CalleeSaveFrameSize(type),
X86CalleeSaveCoreSpills(type),
X86CalleeSaveFpSpills(type));
diff --git a/runtime/arch/x86_64/asm_support_x86_64.h b/runtime/arch/x86_64/asm_support_x86_64.h
index a4446d3..51befbe 100644
--- a/runtime/arch/x86_64/asm_support_x86_64.h
+++ b/runtime/arch/x86_64/asm_support_x86_64.h
@@ -23,5 +23,7 @@
#define FRAME_SIZE_SAVE_REFS_ONLY (64 + 4*8)
#define FRAME_SIZE_SAVE_REFS_AND_ARGS (112 + 12*8)
#define FRAME_SIZE_SAVE_EVERYTHING (144 + 16*8)
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING
+#define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING
#endif // ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_H_
diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
index 0a9199e..73e8610 100644
--- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S
+++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
@@ -272,7 +272,7 @@
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
* when R14 and R15 are already saved.
*/
-MACRO0(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED)
+MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
#if defined(__APPLE__)
int3
int3
@@ -316,7 +316,7 @@
movq %xmm14, 120(%rsp)
movq %xmm15, 128(%rsp)
// Push ArtMethod* for save everything frame method.
- pushq RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET(%r10)
+ pushq \runtime_method_offset(%r10)
CFI_ADJUST_CFA_OFFSET(8)
// Store rsp as the top quick frame.
movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET
@@ -334,18 +334,18 @@
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
* when R15 is already saved.
*/
-MACRO0(SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED)
+MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
PUSH r14
- SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED
+ SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED \runtime_method_offset
END_MACRO
/*
* Macro that sets up the callee save frame to conform with
* Runtime::CreateCalleeSaveMethod(kSaveEverything)
*/
-MACRO0(SETUP_SAVE_EVERYTHING_FRAME)
+MACRO1(SETUP_SAVE_EVERYTHING_FRAME, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
PUSH r15
- SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED
+ SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED \runtime_method_offset
END_MACRO
MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_FRPS)
@@ -951,9 +951,9 @@
END_MACRO
// Macro for string and type resolution and initialization.
-MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name)
+MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
DEFINE_FUNCTION VAR(c_name)
- SETUP_SAVE_EVERYTHING_FRAME // save everything for GC
+ SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for GC
// Outgoing argument set up
movl %eax, %edi // pass string index
movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current()
@@ -970,6 +970,10 @@
END_FUNCTION VAR(c_name)
END_MACRO
+MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name)
+ ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET
+END_MACRO
+
MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER)
testq %rax, %rax // rax == 0 ?
jz 1f // if rax == 0 goto 1
@@ -1290,8 +1294,8 @@
ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedRegionTLAB
END_FUNCTION art_quick_alloc_object_initialized_region_tlab
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
-ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
+ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode
ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode
@@ -1575,7 +1579,7 @@
END_FUNCTION art_quick_memcpy
DEFINE_FUNCTION art_quick_test_suspend
- SETUP_SAVE_EVERYTHING_FRAME // save everything for GC
+ SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save everything for GC
// Outgoing argument set up
movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current()
call SYMBOL(artTestSuspendFromCode) // (Thread*)
diff --git a/runtime/arch/x86_64/quick_method_frame_info_x86_64.h b/runtime/arch/x86_64/quick_method_frame_info_x86_64.h
index 425d616..ebf976e 100644
--- a/runtime/arch/x86_64/quick_method_frame_info_x86_64.h
+++ b/runtime/arch/x86_64/quick_method_frame_info_x86_64.h
@@ -56,24 +56,28 @@
(1 << art::x86_64::XMM10) | (1 << art::x86_64::XMM11);
constexpr uint32_t X86_64CalleeSaveCoreSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kX86_64CalleeSaveAlwaysSpills | kX86_64CalleeSaveRefSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kX86_64CalleeSaveArgSpills : 0) |
(type == CalleeSaveType::kSaveEverything ? kX86_64CalleeSaveEverythingSpills : 0);
}
constexpr uint32_t X86_64CalleeSaveFpSpills(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return kX86_64CalleeSaveFpSpills |
(type == CalleeSaveType::kSaveRefsAndArgs ? kX86_64CalleeSaveFpArgSpills : 0) |
(type == CalleeSaveType::kSaveEverything ? kX86_64CalleeSaveFpEverythingSpills : 0);
}
constexpr uint32_t X86_64CalleeSaveFrameSize(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return RoundUp((POPCOUNT(X86_64CalleeSaveCoreSpills(type)) /* gprs */ +
POPCOUNT(X86_64CalleeSaveFpSpills(type)) /* fprs */ +
1 /* Method* */) * static_cast<size_t>(kX86_64PointerSize), kStackAlignment);
}
constexpr QuickMethodFrameInfo X86_64CalleeSaveMethodFrameInfo(CalleeSaveType type) {
+ type = GetCanonicalCalleeSaveType(type);
return QuickMethodFrameInfo(X86_64CalleeSaveFrameSize(type),
X86_64CalleeSaveCoreSpills(type),
X86_64CalleeSaveFpSpills(type));
diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h
index fad9278..50e9144 100644
--- a/runtime/art_method-inl.h
+++ b/runtime/art_method-inl.h
@@ -285,6 +285,10 @@
return "<runtime internal callee-save reference and argument registers method>";
} else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything)) {
return "<runtime internal save-every-register method>";
+ } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit)) {
+ return "<runtime internal save-every-register method for clinit>";
+ } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck)) {
+ return "<runtime internal save-every-register method for suspend check>";
} else {
return "<unknown runtime internal method>";
}
diff --git a/runtime/base/callee_save_type.h b/runtime/base/callee_save_type.h
index 501b296..e9cd63c 100644
--- a/runtime/base/callee_save_type.h
+++ b/runtime/base/callee_save_type.h
@@ -28,10 +28,20 @@
kSaveRefsOnly, // Only those callee-save registers that can hold references.
kSaveRefsAndArgs, // References (see above) and arguments (usually caller-save registers).
kSaveEverything, // All registers, including both callee-save and caller-save.
+ kSaveEverythingForClinit, // Special kSaveEverything for clinit.
+ kSaveEverythingForSuspendCheck, // Special kSaveEverything for suspend check.
kLastCalleeSaveType // Value used for iteration.
};
std::ostream& operator<<(std::ostream& os, const CalleeSaveType& rhs);
+static inline constexpr CalleeSaveType GetCanonicalCalleeSaveType(CalleeSaveType type) {
+ if (type == CalleeSaveType::kSaveEverythingForClinit ||
+ type == CalleeSaveType::kSaveEverythingForSuspendCheck) {
+ return CalleeSaveType::kSaveEverything;
+ }
+ return type;
+}
+
} // namespace art
#endif // ART_RUNTIME_BASE_CALLEE_SAVE_TYPE_H_
diff --git a/runtime/base/unix_file/fd_file.cc b/runtime/base/unix_file/fd_file.cc
index 0c73ce7..eb8ced0 100644
--- a/runtime/base/unix_file/fd_file.cc
+++ b/runtime/base/unix_file/fd_file.cc
@@ -438,4 +438,30 @@
return true;
}
+int FdFile::Compare(FdFile* other) {
+ int64_t length = GetLength();
+ int64_t length2 = other->GetLength();
+ if (length != length2) {
+ return length < length2 ? -1 : 1;
+ }
+ static const size_t kBufferSize = 4096;
+ std::unique_ptr<uint8_t[]> buffer1(new uint8_t[kBufferSize]);
+ std::unique_ptr<uint8_t[]> buffer2(new uint8_t[kBufferSize]);
+ while (length > 0) {
+ size_t len = std::min(kBufferSize, static_cast<size_t>(length));
+ if (!ReadFully(&buffer1[0], len)) {
+ return -1;
+ }
+ if (!other->ReadFully(&buffer2[0], len)) {
+ return 1;
+ }
+ int result = memcmp(&buffer1[0], &buffer2[0], len);
+ if (result != 0) {
+ return result;
+ }
+ length -= len;
+ }
+ return 0;
+}
+
} // namespace unix_file
diff --git a/runtime/base/unix_file/fd_file.h b/runtime/base/unix_file/fd_file.h
index e07c3fd..91b08bc 100644
--- a/runtime/base/unix_file/fd_file.h
+++ b/runtime/base/unix_file/fd_file.h
@@ -145,6 +145,11 @@
// WARNING: Only use this when you know what you're doing!
void MarkUnchecked();
+ // Compare against another file. Returns 0 if the files are equivalent, otherwise returns -1 or 1
+ // depending on if the lenghts are different. If the lengths are the same, the function returns
+ // the difference of the first byte that differs.
+ int Compare(FdFile* other);
+
protected:
// If the guard state indicates checking (!=kNoCheck), go to the target state "target". Print the
// given warning if the current state is or exceeds warn_threshold.
diff --git a/runtime/base/unix_file/fd_file_test.cc b/runtime/base/unix_file/fd_file_test.cc
index 6aef348..8b1a115 100644
--- a/runtime/base/unix_file/fd_file_test.cc
+++ b/runtime/base/unix_file/fd_file_test.cc
@@ -220,4 +220,58 @@
EXPECT_FALSE(art::OS::FileExists(filename.c_str())) << filename;
}
+TEST_F(FdFileTest, Compare) {
+ std::vector<uint8_t> buffer;
+ constexpr int64_t length = 17 * art::KB;
+ for (size_t i = 0; i < length; ++i) {
+ buffer.push_back(static_cast<uint8_t>(i));
+ }
+
+ auto reset_compare = [&](art::ScratchFile& a, art::ScratchFile& b) {
+ a.GetFile()->ResetOffset();
+ b.GetFile()->ResetOffset();
+ return a.GetFile()->Compare(b.GetFile());
+ };
+
+ art::ScratchFile tmp;
+ EXPECT_TRUE(tmp.GetFile()->WriteFully(&buffer[0], length));
+ EXPECT_EQ(tmp.GetFile()->GetLength(), length);
+
+ art::ScratchFile tmp2;
+ EXPECT_TRUE(tmp2.GetFile()->WriteFully(&buffer[0], length));
+ EXPECT_EQ(tmp2.GetFile()->GetLength(), length);
+
+ // Basic equality check.
+ tmp.GetFile()->ResetOffset();
+ tmp2.GetFile()->ResetOffset();
+ // Files should be the same.
+ EXPECT_EQ(reset_compare(tmp, tmp2), 0);
+
+ // Change a byte near the start.
+ ++buffer[2];
+ art::ScratchFile tmp3;
+ EXPECT_TRUE(tmp3.GetFile()->WriteFully(&buffer[0], length));
+ --buffer[2];
+ EXPECT_NE(reset_compare(tmp, tmp3), 0);
+
+ // Change a byte near the middle.
+ ++buffer[length / 2];
+ art::ScratchFile tmp4;
+ EXPECT_TRUE(tmp4.GetFile()->WriteFully(&buffer[0], length));
+ --buffer[length / 2];
+ EXPECT_NE(reset_compare(tmp, tmp4), 0);
+
+ // Change a byte near the end.
+ ++buffer[length - 5];
+ art::ScratchFile tmp5;
+ EXPECT_TRUE(tmp5.GetFile()->WriteFully(&buffer[0], length));
+ --buffer[length - 5];
+ EXPECT_NE(reset_compare(tmp, tmp5), 0);
+
+ // Reference check
+ art::ScratchFile tmp6;
+ EXPECT_TRUE(tmp6.GetFile()->WriteFully(&buffer[0], length));
+ EXPECT_EQ(reset_compare(tmp, tmp6), 0);
+}
+
} // namespace unix_file
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 3ac87c5..9756f57 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -3694,7 +3694,8 @@
ObjPtr<mirror::DexCache> ClassLinker::FindDexCache(Thread* self, const DexFile& dex_file) {
ReaderMutexLock mu(self, *Locks::dex_lock_);
- ObjPtr<mirror::DexCache> dex_cache = DecodeDexCache(self, FindDexCacheDataLocked(dex_file));
+ DexCacheData dex_cache_data = FindDexCacheDataLocked(dex_file);
+ ObjPtr<mirror::DexCache> dex_cache = DecodeDexCache(self, dex_cache_data);
if (dex_cache != nullptr) {
return dex_cache;
}
@@ -3704,7 +3705,8 @@
LOG(FATAL_WITHOUT_ABORT) << "Registered dex file " << data.dex_file->GetLocation();
}
}
- LOG(FATAL) << "Failed to find DexCache for DexFile " << dex_file.GetLocation();
+ LOG(FATAL) << "Failed to find DexCache for DexFile " << dex_file.GetLocation()
+ << " " << &dex_file << " " << dex_cache_data.dex_file;
UNREACHABLE();
}
@@ -4280,13 +4282,7 @@
std::string error_msg;
verifier::FailureKind verifier_failure = verifier::FailureKind::kNoFailure;
if (!preverified) {
- Runtime* runtime = Runtime::Current();
- verifier_failure = verifier::MethodVerifier::VerifyClass(self,
- klass.Get(),
- runtime->GetCompilerCallbacks(),
- runtime->IsAotCompiler(),
- log_level,
- &error_msg);
+ verifier_failure = PerformClassVerification(self, klass, log_level, &error_msg);
}
// Verification is done, grab the lock again.
@@ -4354,6 +4350,19 @@
return verifier_failure;
}
+verifier::FailureKind ClassLinker::PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg) {
+ Runtime* const runtime = Runtime::Current();
+ return verifier::MethodVerifier::VerifyClass(self,
+ klass.Get(),
+ runtime->GetCompilerCallbacks(),
+ runtime->IsAotCompiler(),
+ log_level,
+ error_msg);
+}
+
bool ClassLinker::VerifyClassUsingOatFile(const DexFile& dex_file,
ObjPtr<mirror::Class> klass,
mirror::Class::Status& oat_file_class_status) {
@@ -7119,7 +7128,7 @@
method_alignment_);
const size_t old_methods_ptr_size = (old_methods != nullptr) ? old_size : 0;
auto* methods = reinterpret_cast<LengthPrefixedArray<ArtMethod>*>(
- Runtime::Current()->GetLinearAlloc()->Realloc(
+ class_linker_->GetAllocatorForClassLoader(klass_->GetClassLoader())->Realloc(
self_, old_methods, old_methods_ptr_size, new_size));
CHECK(methods != nullptr); // Native allocation failure aborts.
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index bf14aeb..2fbbe79 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -710,6 +710,12 @@
REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!Locks::dex_lock_);
+ virtual verifier::FailureKind PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
private:
class LinkInterfaceMethodsHelper;
diff --git a/runtime/class_loader_context.cc b/runtime/class_loader_context.cc
index b50aec0..e7051b3 100644
--- a/runtime/class_loader_context.cc
+++ b/runtime/class_loader_context.cc
@@ -68,6 +68,10 @@
}
}
+std::unique_ptr<ClassLoaderContext> ClassLoaderContext::Default() {
+ return Create("");
+}
+
std::unique_ptr<ClassLoaderContext> ClassLoaderContext::Create(const std::string& spec) {
std::unique_ptr<ClassLoaderContext> result(new ClassLoaderContext());
if (result->Parse(spec)) {
diff --git a/runtime/class_loader_context.h b/runtime/class_loader_context.h
index 85299d4..9afa880 100644
--- a/runtime/class_loader_context.h
+++ b/runtime/class_loader_context.h
@@ -34,9 +34,6 @@
// Utility class which holds the class loader context used during compilation/verification.
class ClassLoaderContext {
public:
- // Creates an empty context (with no class loaders).
- ClassLoaderContext();
-
~ClassLoaderContext();
// Opens requested class path files and appends them to ClassLoaderInfo::opened_dex_files.
@@ -126,6 +123,10 @@
static std::unique_ptr<ClassLoaderContext> CreateContextForClassLoader(jobject class_loader,
jobjectArray dex_elements);
+ // Returns the default class loader context to be used when none is specified.
+ // This will return a context with a single and empty PathClassLoader.
+ static std::unique_ptr<ClassLoaderContext> Default();
+
private:
enum ClassLoaderType {
kInvalidClassLoader = 0,
@@ -151,6 +152,9 @@
explicit ClassLoaderInfo(ClassLoaderType cl_type) : type(cl_type) {}
};
+ // Creates an empty context (with no class loaders).
+ ClassLoaderContext();
+
// Constructs an empty context.
// `owns_the_dex_files` specifies whether or not the context will own the opened dex files
// present in the class loader chain. If `owns_the_dex_files` is true then OpenDexFiles cannot
diff --git a/runtime/compiler_callbacks.h b/runtime/compiler_callbacks.h
index 806653a..c51bb5e 100644
--- a/runtime/compiler_callbacks.h
+++ b/runtime/compiler_callbacks.h
@@ -22,6 +22,8 @@
namespace art {
+class CompilerDriver;
+
namespace verifier {
class MethodVerifier;
@@ -49,6 +51,13 @@
virtual verifier::VerifierDeps* GetVerifierDeps() const = 0;
virtual void SetVerifierDeps(verifier::VerifierDeps* deps ATTRIBUTE_UNUSED) {}
+ virtual bool CanAssumeVerified(ClassReference ref ATTRIBUTE_UNUSED) {
+ return false;
+ }
+
+ virtual void SetDoesClassUnloading(bool does_class_unloading ATTRIBUTE_UNUSED,
+ CompilerDriver* compiler_driver ATTRIBUTE_UNUSED) {}
+
bool IsBootImage() {
return mode_ == CallbackMode::kCompileBootImage;
}
diff --git a/runtime/dex_file_verifier.cc b/runtime/dex_file_verifier.cc
index c5c4eda..8fdd470 100644
--- a/runtime/dex_file_verifier.cc
+++ b/runtime/dex_file_verifier.cc
@@ -363,7 +363,7 @@
// Check file size from the header.
uint32_t expected_size = header_->file_size_;
if (size_ != expected_size) {
- ErrorStringPrintf("Bad file size (%zd, expected %ud)", size_, expected_size);
+ ErrorStringPrintf("Bad file size (%zd, expected %u)", size_, expected_size);
return false;
}
diff --git a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc
index a2a6e08..5355267 100644
--- a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc
@@ -65,8 +65,8 @@
// A class may be accessing another class' fields when it doesn't have access, as access has been
// given by inheritance.
ScopedQuickEntrypointChecks sqec(self);
- auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self,
- CalleeSaveType::kSaveEverything);
+ auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(
+ self, CalleeSaveType::kSaveEverythingForClinit);
ArtMethod* caller = caller_and_outer.caller;
mirror::Class* result =
ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, true, false);
@@ -80,8 +80,8 @@
REQUIRES_SHARED(Locks::mutator_lock_) {
// Called when method->dex_cache_resolved_types_[] misses.
ScopedQuickEntrypointChecks sqec(self);
- auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self,
- CalleeSaveType::kSaveEverything);
+ auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(
+ self, CalleeSaveType::kSaveEverythingForClinit);
ArtMethod* caller = caller_and_outer.caller;
mirror::Class* result =
ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, false, false);
diff --git a/runtime/entrypoints/quick/quick_field_entrypoints.cc b/runtime/entrypoints/quick/quick_field_entrypoints.cc
index 726bddd..b298db6 100644
--- a/runtime/entrypoints/quick/quick_field_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_field_entrypoints.cc
@@ -301,6 +301,7 @@
extern "C" mirror::Object* artReadBarrierSlow(mirror::Object* ref ATTRIBUTE_UNUSED,
mirror::Object* obj,
uint32_t offset) {
+ // Used only in connection with non-volatile loads.
DCHECK(kEmitCompilerReadBarrier);
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(obj) + offset;
mirror::HeapReference<mirror::Object>* ref_addr =
@@ -308,9 +309,10 @@
constexpr ReadBarrierOption kReadBarrierOption =
kUseReadBarrier ? kWithReadBarrier : kWithoutReadBarrier;
mirror::Object* result =
- ReadBarrier::Barrier<mirror::Object, kReadBarrierOption>(obj,
- MemberOffset(offset),
- ref_addr);
+ ReadBarrier::Barrier<mirror::Object, /* kIsVolatile */ false, kReadBarrierOption>(
+ obj,
+ MemberOffset(offset),
+ ref_addr);
return result;
}
diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc
index 7e08b7a..b692618 100644
--- a/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc
+++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc
@@ -90,7 +90,18 @@
GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveRefsOnly)); \
CheckFrameSize(isa, \
CalleeSaveType::kSaveAllCalleeSaves, \
- GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveAllCalleeSaves))
+ GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveAllCalleeSaves)); \
+ CheckFrameSize(isa, \
+ CalleeSaveType::kSaveEverything, \
+ GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveEverything)); \
+ CheckFrameSize(isa, \
+ CalleeSaveType::kSaveEverythingForClinit, \
+ GetCalleeSaveFrameSize(isa, \
+ CalleeSaveType::kSaveEverythingForClinit)); \
+ CheckFrameSize(isa, \
+ CalleeSaveType::kSaveEverythingForSuspendCheck, \
+ GetCalleeSaveFrameSize( \
+ isa, CalleeSaveType::kSaveEverythingForSuspendCheck))
CHECK_FRAME_SIZE(kArm);
CHECK_FRAME_SIZE(kArm64);
@@ -123,6 +134,13 @@
GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveRefsOnly));
CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveAllCalleeSaves,
GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveAllCalleeSaves));
+ CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveEverything,
+ GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveEverything));
+ CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveEverythingForClinit,
+ GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveEverythingForClinit));
+ CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveEverythingForSuspendCheck,
+ GetCalleeSaveReturnPcOffset(kRuntimeISA,
+ CalleeSaveType::kSaveEverythingForSuspendCheck));
}
} // namespace art
diff --git a/runtime/fault_handler.cc b/runtime/fault_handler.cc
index fd0cd5f..7d01af0 100644
--- a/runtime/fault_handler.cc
+++ b/runtime/fault_handler.cc
@@ -79,8 +79,7 @@
static mirror::Class* SafeGetClass(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
char* obj_cls = reinterpret_cast<char*>(obj) + mirror::Object::ClassOffset().SizeValue();
- mirror::HeapReference<mirror::Class> cls =
- mirror::HeapReference<mirror::Class>::FromMirrorPtr(nullptr);
+ mirror::HeapReference<mirror::Class> cls;
ssize_t rc = SafeCopy(&cls, obj_cls, sizeof(cls));
CHECK_NE(-1, rc);
diff --git a/runtime/gc/accounting/mod_union_table.cc b/runtime/gc/accounting/mod_union_table.cc
index 2901995..1b3d0da 100644
--- a/runtime/gc/accounting/mod_union_table.cc
+++ b/runtime/gc/accounting/mod_union_table.cc
@@ -115,8 +115,8 @@
}
private:
- template<bool kPoisonReferences>
- void MarkReference(mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) const
+ template<typename CompressedReferenceType>
+ void MarkReference(CompressedReferenceType* obj_ptr) const
REQUIRES_SHARED(Locks::mutator_lock_) {
// Only add the reference if it is non null and fits our criteria.
mirror::Object* ref = obj_ptr->AsMirrorPtr();
diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc
index 92b4360..280c0b1 100644
--- a/runtime/gc/accounting/space_bitmap.cc
+++ b/runtime/gc/accounting/space_bitmap.cc
@@ -48,16 +48,21 @@
CHECK(mem_map != nullptr);
uintptr_t* bitmap_begin = reinterpret_cast<uintptr_t*>(mem_map->Begin());
const size_t bitmap_size = ComputeBitmapSize(heap_capacity);
- return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin);
+ return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin, heap_capacity);
}
template<size_t kAlignment>
-SpaceBitmap<kAlignment>::SpaceBitmap(const std::string& name, MemMap* mem_map, uintptr_t* bitmap_begin,
- size_t bitmap_size, const void* heap_begin)
+SpaceBitmap<kAlignment>::SpaceBitmap(const std::string& name,
+ MemMap* mem_map,
+ uintptr_t* bitmap_begin,
+ size_t bitmap_size,
+ const void* heap_begin,
+ size_t heap_capacity)
: mem_map_(mem_map),
bitmap_begin_(reinterpret_cast<Atomic<uintptr_t>*>(bitmap_begin)),
bitmap_size_(bitmap_size),
heap_begin_(reinterpret_cast<uintptr_t>(heap_begin)),
+ heap_limit_(reinterpret_cast<uintptr_t>(heap_begin) + heap_capacity),
name_(name) {
CHECK(bitmap_begin_ != nullptr);
CHECK_NE(bitmap_size, 0U);
@@ -89,6 +94,7 @@
if (new_size < bitmap_size_) {
bitmap_size_ = new_size;
}
+ heap_limit_ = new_end;
// Not sure if doing this trim is necessary, since nothing past the end of the heap capacity
// should be marked.
}
diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h
index 2fe6394..b49e0b7 100644
--- a/runtime/gc/accounting/space_bitmap.h
+++ b/runtime/gc/accounting/space_bitmap.h
@@ -163,6 +163,7 @@
void SetHeapSize(size_t bytes) {
// TODO: Un-map the end of the mem map.
+ heap_limit_ = heap_begin_ + bytes;
bitmap_size_ = OffsetToIndex(bytes) * sizeof(intptr_t);
CHECK_EQ(HeapSize(), bytes);
}
@@ -173,7 +174,7 @@
// The maximum address which the bitmap can span. (HeapBegin() <= object < HeapLimit()).
uint64_t HeapLimit() const {
- return static_cast<uint64_t>(HeapBegin()) + HeapSize();
+ return heap_limit_;
}
// Set the max address which can covered by the bitmap.
@@ -196,8 +197,12 @@
private:
// TODO: heap_end_ is initialized so that the heap bitmap is empty, this doesn't require the -1,
// however, we document that this is expected on heap_end_
- SpaceBitmap(const std::string& name, MemMap* mem_map, uintptr_t* bitmap_begin, size_t bitmap_size,
- const void* heap_begin);
+ SpaceBitmap(const std::string& name,
+ MemMap* mem_map,
+ uintptr_t* bitmap_begin,
+ size_t bitmap_size,
+ const void* heap_begin,
+ size_t heap_capacity);
template<bool kSetBit>
bool Modify(const mirror::Object* obj);
@@ -211,10 +216,13 @@
// Size of this bitmap.
size_t bitmap_size_;
- // The base address of the heap, which corresponds to the word containing the first bit in the
- // bitmap.
+ // The start address of the memory covered by the bitmap, which corresponds to the word
+ // containing the first bit in the bitmap.
const uintptr_t heap_begin_;
+ // The end address of the memory covered by the bitmap. This may not be on a word boundary.
+ uintptr_t heap_limit_;
+
// Name of this bitmap.
std::string name_;
};
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 9d672b1..52b355d 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -2431,24 +2431,31 @@
// Non-immune non-moving space. Use the mark bitmap.
accounting::ContinuousSpaceBitmap* mark_bitmap =
heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref);
- accounting::LargeObjectBitmap* los_bitmap =
- heap_mark_bitmap_->GetLargeObjectBitmap(from_ref);
- CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range";
bool is_los = mark_bitmap == nullptr;
if (!is_los && mark_bitmap->Test(from_ref)) {
// Already marked.
to_ref = from_ref;
- } else if (is_los && los_bitmap->Test(from_ref)) {
- // Already marked in LOS.
- to_ref = from_ref;
} else {
- // Not marked.
- if (IsOnAllocStack(from_ref)) {
- // If on the allocation stack, it's considered marked.
+ accounting::LargeObjectBitmap* los_bitmap =
+ heap_mark_bitmap_->GetLargeObjectBitmap(from_ref);
+ // We may not have a large object space for dex2oat, don't assume it exists.
+ if (los_bitmap == nullptr) {
+ CHECK(heap_->GetLargeObjectsSpace() == nullptr)
+ << "LOS bitmap covers the entire address range " << from_ref
+ << " " << heap_->DumpSpaces();
+ }
+ if (los_bitmap != nullptr && is_los && los_bitmap->Test(from_ref)) {
+ // Already marked in LOS.
to_ref = from_ref;
} else {
// Not marked.
- to_ref = nullptr;
+ if (IsOnAllocStack(from_ref)) {
+ // If on the allocation stack, it's considered marked.
+ to_ref = from_ref;
+ } else {
+ // Not marked.
+ to_ref = nullptr;
+ }
}
}
}
@@ -2457,6 +2464,7 @@
}
bool ConcurrentCopying::IsOnAllocStack(mirror::Object* ref) {
+ // TODO: Explain why this is here. What release operation does it pair with?
QuasiAtomic::ThreadFenceAcquire();
accounting::ObjectStack* alloc_stack = GetAllocationStack();
return alloc_stack->Contains(ref);
@@ -2617,9 +2625,8 @@
}
} while (!field->CasWeakRelaxed(from_ref, to_ref));
} else {
- QuasiAtomic::ThreadFenceRelease();
- field->Assign(to_ref);
- QuasiAtomic::ThreadFenceSequentiallyConsistent();
+ // TODO: Why is this seq_cst when the above is relaxed? Document memory ordering.
+ field->Assign</* kIsVolatile */ true>(to_ref);
}
}
return true;
diff --git a/runtime/gc/collector/garbage_collector.h b/runtime/gc/collector/garbage_collector.h
index dec206b..f722e8d 100644
--- a/runtime/gc/collector/garbage_collector.h
+++ b/runtime/gc/collector/garbage_collector.h
@@ -113,7 +113,7 @@
virtual mirror::Object* IsMarked(mirror::Object* obj)
REQUIRES_SHARED(Locks::mutator_lock_) = 0;
// Returns true if the given heap reference is null or is already marked. If it's already marked,
- // update the reference (uses a CAS if do_atomic_update is true. Otherwise, returns false.
+ // update the reference (uses a CAS if do_atomic_update is true). Otherwise, returns false.
virtual bool IsNullOrMarkedHeapReference(mirror::HeapReference<mirror::Object>* obj,
bool do_atomic_update)
REQUIRES_SHARED(Locks::mutator_lock_) = 0;
diff --git a/runtime/gc/collector/semi_space-inl.h b/runtime/gc/collector/semi_space-inl.h
index 78fb2d2..7db5d2c 100644
--- a/runtime/gc/collector/semi_space-inl.h
+++ b/runtime/gc/collector/semi_space-inl.h
@@ -38,9 +38,8 @@
// Used to mark and copy objects. Any newly-marked objects who are in the from space Get moved to
// the to-space and have their forward address updated. Objects which have been newly marked are
// pushed on the mark stack.
-template<bool kPoisonReferences>
-inline void SemiSpace::MarkObject(
- mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) {
+template<typename CompressedReferenceType>
+inline void SemiSpace::MarkObject(CompressedReferenceType* obj_ptr) {
mirror::Object* obj = obj_ptr->AsMirrorPtr();
if (obj == nullptr) {
return;
@@ -73,9 +72,8 @@
}
}
-template<bool kPoisonReferences>
-inline void SemiSpace::MarkObjectIfNotInToSpace(
- mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr) {
+template<typename CompressedReferenceType>
+inline void SemiSpace::MarkObjectIfNotInToSpace(CompressedReferenceType* obj_ptr) {
if (!to_space_->HasAddress(obj_ptr->AsMirrorPtr())) {
MarkObject(obj_ptr);
}
diff --git a/runtime/gc/collector/semi_space.h b/runtime/gc/collector/semi_space.h
index fd52da3..6d4d789 100644
--- a/runtime/gc/collector/semi_space.h
+++ b/runtime/gc/collector/semi_space.h
@@ -97,13 +97,13 @@
// Find the default mark bitmap.
void FindDefaultMarkBitmap();
- // Updates obj_ptr if the object has moved.
- template<bool kPoisonReferences>
- void MarkObject(mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr)
+ // Updates obj_ptr if the object has moved. Takes either an ObjectReference or a HeapReference.
+ template<typename CompressedReferenceType>
+ void MarkObject(CompressedReferenceType* obj_ptr)
REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_);
- template<bool kPoisonReferences>
- void MarkObjectIfNotInToSpace(mirror::ObjectReference<kPoisonReferences, mirror::Object>* obj_ptr)
+ template<typename CompressedReferenceType>
+ void MarkObjectIfNotInToSpace(CompressedReferenceType* obj_ptr)
REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_);
virtual mirror::Object* MarkObject(mirror::Object* root) OVERRIDE
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index 1484382..14e017a 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -725,6 +725,10 @@
image_header->GetImageMethod(ImageHeader::kSaveRefsAndArgsMethod));
CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything),
image_header->GetImageMethod(ImageHeader::kSaveEverythingMethod));
+ CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit),
+ image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForClinit));
+ CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck),
+ image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForSuspendCheck));
} else if (!runtime->HasResolutionMethod()) {
runtime->SetInstructionSet(space->oat_file_non_owned_->GetOatHeader().GetInstructionSet());
runtime->SetResolutionMethod(image_header->GetImageMethod(ImageHeader::kResolutionMethod));
@@ -743,6 +747,12 @@
runtime->SetCalleeSaveMethod(
image_header->GetImageMethod(ImageHeader::kSaveEverythingMethod),
CalleeSaveType::kSaveEverything);
+ runtime->SetCalleeSaveMethod(
+ image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForClinit),
+ CalleeSaveType::kSaveEverythingForClinit);
+ runtime->SetCalleeSaveMethod(
+ image_header->GetImageMethod(ImageHeader::kSaveEverythingMethodForSuspendCheck),
+ CalleeSaveType::kSaveEverythingForSuspendCheck);
}
VLOG(image) << "ImageSpace::Init exiting " << *space.get();
diff --git a/runtime/gc/space/large_object_space_test.cc b/runtime/gc/space/large_object_space_test.cc
index 79b775a..9baa016 100644
--- a/runtime/gc/space/large_object_space_test.cc
+++ b/runtime/gc/space/large_object_space_test.cc
@@ -38,12 +38,19 @@
Thread* const self = Thread::Current();
for (size_t i = 0; i < 2; ++i) {
LargeObjectSpace* los = nullptr;
+ const size_t capacity = 128 * MB;
if (i == 0) {
los = space::LargeObjectMapSpace::Create("large object space");
} else {
- los = space::FreeListSpace::Create("large object space", nullptr, 128 * MB);
+ los = space::FreeListSpace::Create("large object space", nullptr, capacity);
}
+ // Make sure the bitmap is not empty and actually covers at least how much we expect.
+ CHECK_LT(static_cast<uintptr_t>(los->GetLiveBitmap()->HeapBegin()),
+ static_cast<uintptr_t>(los->GetLiveBitmap()->HeapLimit()));
+ CHECK_LE(static_cast<uintptr_t>(los->GetLiveBitmap()->HeapBegin() + capacity),
+ static_cast<uintptr_t>(los->GetLiveBitmap()->HeapLimit()));
+
static const size_t num_allocations = 64;
static const size_t max_allocation_size = 0x100000;
std::vector<std::pair<mirror::Object*, size_t>> requests;
diff --git a/runtime/gc/space/malloc_space.cc b/runtime/gc/space/malloc_space.cc
index c2a8de3..c994127 100644
--- a/runtime/gc/space/malloc_space.cc
+++ b/runtime/gc/space/malloc_space.cc
@@ -191,14 +191,10 @@
VLOG(heap) << "Size " << GetMemMap()->Size();
VLOG(heap) << "GrowthLimit " << PrettySize(growth_limit);
VLOG(heap) << "Capacity " << PrettySize(capacity);
- // Remap the tail. Pass MAP_PRIVATE since we don't want to share the same ashmem as the zygote
- // space.
+ // Remap the tail.
std::string error_msg;
- std::unique_ptr<MemMap> mem_map(GetMemMap()->RemapAtEnd(End(),
- alloc_space_name,
- PROT_READ | PROT_WRITE,
- MAP_PRIVATE,
- &error_msg));
+ std::unique_ptr<MemMap> mem_map(GetMemMap()->RemapAtEnd(End(), alloc_space_name,
+ PROT_READ | PROT_WRITE, &error_msg));
CHECK(mem_map.get() != nullptr) << error_msg;
void* allocator = CreateAllocator(End(), starting_size_, initial_size_, capacity,
low_memory_mode);
diff --git a/runtime/generated/asm_support_gen.h b/runtime/generated/asm_support_gen.h
index 11b3abb..314c45e 100644
--- a/runtime/generated/asm_support_gen.h
+++ b/runtime/generated/asm_support_gen.h
@@ -34,6 +34,10 @@
DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveRefsAndArgs))))
#define RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 0x18
DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveEverything))))
+#define RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET 0x20
+DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveEverythingForClinit))))
+#define RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET 0x28
+DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveEverythingForSuspendCheck))))
#define THREAD_FLAGS_OFFSET 0
DEFINE_CHECK_EQ(static_cast<int32_t>(THREAD_FLAGS_OFFSET), (static_cast<int32_t>(art::Thread:: ThreadFlagsOffset<art::kRuntimePointerSize>().Int32Value())))
#define THREAD_ID_OFFSET 12
diff --git a/runtime/image.h b/runtime/image.h
index 6c76f49..7bb796c 100644
--- a/runtime/image.h
+++ b/runtime/image.h
@@ -183,6 +183,8 @@
kSaveRefsOnlyMethod,
kSaveRefsAndArgsMethod,
kSaveEverythingMethod,
+ kSaveEverythingMethodForClinit,
+ kSaveEverythingMethodForSuspendCheck,
kImageMethodsCount, // Number of elements in enum.
};
diff --git a/runtime/interpreter/interpreter_common.h b/runtime/interpreter/interpreter_common.h
index b228e28..5b942f2 100644
--- a/runtime/interpreter/interpreter_common.h
+++ b/runtime/interpreter/interpreter_common.h
@@ -141,11 +141,8 @@
return false;
} else {
jit::Jit* jit = Runtime::Current()->GetJit();
- if (jit != nullptr) {
- if (type == kVirtual) {
- jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method);
- }
- jit->AddSamples(self, sf_method, 1, /*with_backedges*/false);
+ if (jit != nullptr && type == kVirtual) {
+ jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method);
}
if (called_method->IsIntrinsic()) {
if (MterpHandleIntrinsic(&shadow_frame, called_method, inst, inst_data,
@@ -182,11 +179,8 @@
return false;
} else {
jit::Jit* jit = Runtime::Current()->GetJit();
- if (jit != nullptr) {
- if (type == kVirtual || type == kInterface) {
- jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method);
- }
- jit->AddSamples(self, sf_method, 1, /*with_backedges*/false);
+ if (jit != nullptr && (type == kVirtual || type == kInterface)) {
+ jit->InvokeVirtualOrInterface(receiver, sf_method, shadow_frame.GetDexPC(), called_method);
}
// TODO: Remove the InvokeVirtualOrInterface instrumentation, as it was only used by the JIT.
if (type == kVirtual || type == kInterface) {
diff --git a/runtime/interpreter/mterp/mterp.cc b/runtime/interpreter/mterp/mterp.cc
index 5955b90..88254a8 100644
--- a/runtime/interpreter/mterp/mterp.cc
+++ b/runtime/interpreter/mterp/mterp.cc
@@ -280,7 +280,6 @@
if (jit != nullptr) {
jit->InvokeVirtualOrInterface(
receiver, shadow_frame->GetMethod(), shadow_frame->GetDexPC(), called_method);
- jit->AddSamples(self, shadow_frame->GetMethod(), 1, /*with_backedges*/false);
}
return !self->IsExceptionPending();
}
diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc
index 2c72821..ce06a03 100644
--- a/runtime/interpreter/unstarted_runtime.cc
+++ b/runtime/interpreter/unstarted_runtime.cc
@@ -1438,7 +1438,11 @@
mirror::HeapReference<mirror::Object>* field_addr =
reinterpret_cast<mirror::HeapReference<mirror::Object>*>(
reinterpret_cast<uint8_t*>(obj) + static_cast<size_t>(offset));
- ReadBarrier::Barrier<mirror::Object, kWithReadBarrier, /* kAlwaysUpdateField */ true>(
+ ReadBarrier::Barrier<
+ mirror::Object,
+ /* kIsVolatile */ false,
+ kWithReadBarrier,
+ /* kAlwaysUpdateField */ true>(
obj,
MemberOffset(offset),
field_addr);
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 59373eb..47ace7f 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -47,13 +47,9 @@
static constexpr int kProtAll = PROT_READ | PROT_WRITE | PROT_EXEC;
static constexpr int kProtData = PROT_READ | PROT_WRITE;
static constexpr int kProtCode = PROT_READ | PROT_EXEC;
-static constexpr int kProtReadOnly = PROT_READ;
-static constexpr int kProtNone = PROT_NONE;
static constexpr size_t kCodeSizeLogThreshold = 50 * KB;
static constexpr size_t kStackMapSizeLogThreshold = 50 * KB;
-static constexpr size_t kMinMapSpacingPages = 1;
-static constexpr size_t kMaxMapSpacingPages = 128;
#define CHECKED_MPROTECT(memory, size, prot) \
do { \
@@ -64,39 +60,12 @@
} \
} while (false) \
-static MemMap* SplitMemMap(MemMap* existing_map,
- const char* name,
- size_t split_offset,
- int split_prot,
- std::string* error_msg,
- bool use_ashmem,
- unique_fd* shmem_fd = nullptr) {
- std::string error_str;
- uint8_t* divider = existing_map->Begin() + split_offset;
- MemMap* new_map = existing_map->RemapAtEnd(divider,
- name,
- split_prot,
- MAP_SHARED,
- &error_str,
- use_ashmem,
- shmem_fd);
- if (new_map == nullptr) {
- std::ostringstream oss;
- oss << "Failed to create spacing for " << name << ": "
- << error_str << " offset=" << split_offset;
- *error_msg = oss.str();
- return nullptr;
- }
- return new_map;
-}
-
JitCodeCache* JitCodeCache::Create(size_t initial_capacity,
size_t max_capacity,
bool generate_debug_info,
std::string* error_msg) {
ScopedTrace trace(__PRETTY_FUNCTION__);
- CHECK_GT(max_capacity, initial_capacity);
- CHECK_GE(max_capacity - kMaxMapSpacingPages * kPageSize, initial_capacity);
+ CHECK_GE(max_capacity, initial_capacity);
// Generating debug information is for using the Linux perf tool on
// host which does not work with ashmem.
@@ -106,10 +75,6 @@
// With 'perf', we want a 1-1 mapping between an address and a method.
bool garbage_collect_code = !generate_debug_info;
- // We only use two mappings (separating rw from rx) if we are able to use ashmem.
- // See the above comment for debug information and not using ashmem.
- bool use_two_mappings = use_ashmem;
-
// We need to have 32 bit offsets from method headers in code cache which point to things
// in the data cache. If the maps are more than 4G apart, having multiple maps wouldn't work.
// Ensure we're below 1 GB to be safe.
@@ -121,10 +86,6 @@
return nullptr;
}
- // Align both capacities to page size, as that's the unit mspaces use.
- initial_capacity = RoundDown(initial_capacity, 2 * kPageSize);
- max_capacity = RoundDown(max_capacity, 2 * kPageSize);
-
std::string error_str;
// Map name specific for android_os_Debug.cpp accounting.
// Map in low 4gb to simplify accessing root tables for x86_64.
@@ -146,138 +107,35 @@
return nullptr;
}
- // Create a region for JIT data and executable code. This will be
- // laid out as:
- //
- // +----------------+ --------------------
- // | code_sync_map_ | ^ code_sync_size ^
- // | | v |
- // +----------------+ -- |
- // : : ^ |
- // : post_code_map : | post_code_size |
- // : [padding] : v |
- // +----------------+ - |
- // | | ^ |
- // | code_map | | code_size | total_mapping_size
- // | [JIT Code] | v |
- // +----------------+ - |
- // : : ^ |
- // : pre_code_map : | pre_code_size |
- // : [padding] : v |
- // +----------------+ - |
- // | | ^ |
- // | data_map | | data_size |
- // | [Jit Data] | v v
- // +----------------+ --------------------
- //
- // The code_sync_map_ contains a page that we use flush CPU instruction
- // pipelines (see FlushInstructionPipelines()).
- //
- // The padding regions - pre_code_map and post_code_map - exist to
- // put some random distance between the writable JIT code mapping
- // and the executable mapping. The padding is discarded at the end
- // of this function.
- //
- size_t data_size = (max_capacity - kMaxMapSpacingPages * kPageSize) / 2;
- size_t pre_code_size =
- GetRandomNumber(kMinMapSpacingPages, kMaxMapSpacingPages - 1) * kPageSize;
- size_t code_size = max_capacity - data_size - kMaxMapSpacingPages * kPageSize;
- size_t code_sync_size = kPageSize;
- size_t post_code_size = kMaxMapSpacingPages * kPageSize - pre_code_size - code_sync_size;
- DCHECK_EQ(data_size, code_size);
- DCHECK_EQ(pre_code_size + post_code_size + code_sync_size, kMaxMapSpacingPages * kPageSize);
- DCHECK_EQ(data_size + pre_code_size + code_size + post_code_size + code_sync_size, max_capacity);
+ // Align both capacities to page size, as that's the unit mspaces use.
+ initial_capacity = RoundDown(initial_capacity, 2 * kPageSize);
+ max_capacity = RoundDown(max_capacity, 2 * kPageSize);
- // Create pre-code padding region after data region, discarded after
- // code and data regions are set-up.
- std::unique_ptr<MemMap> pre_code_map(SplitMemMap(data_map.get(),
- "jit-code-cache-padding",
- data_size,
- kProtNone,
- error_msg,
- use_ashmem));
- if (pre_code_map == nullptr) {
- return nullptr;
- }
- DCHECK_EQ(data_map->Size(), data_size);
- DCHECK_EQ(pre_code_map->Size(), pre_code_size + code_size + post_code_size + code_sync_size);
+ // Data cache is 1 / 2 of the map.
+ // TODO: Make this variable?
+ size_t data_size = max_capacity / 2;
+ size_t code_size = max_capacity - data_size;
+ DCHECK_EQ(code_size + data_size, max_capacity);
+ uint8_t* divider = data_map->Begin() + data_size;
- // Create code region.
- unique_fd writable_code_fd;
- std::unique_ptr<MemMap> code_map(SplitMemMap(pre_code_map.get(),
- "jit-code-cache",
- pre_code_size,
- use_two_mappings ? kProtCode : kProtAll,
- error_msg,
- use_ashmem,
- &writable_code_fd));
+ MemMap* code_map =
+ data_map->RemapAtEnd(divider, "jit-code-cache", kProtAll, &error_str, use_ashmem);
if (code_map == nullptr) {
+ std::ostringstream oss;
+ oss << "Failed to create read write execute cache: " << error_str << " size=" << max_capacity;
+ *error_msg = oss.str();
return nullptr;
}
- DCHECK_EQ(pre_code_map->Size(), pre_code_size);
- DCHECK_EQ(code_map->Size(), code_size + post_code_size + code_sync_size);
-
- // Padding after code region, discarded after code and data regions
- // are set-up.
- std::unique_ptr<MemMap> post_code_map(SplitMemMap(code_map.get(),
- "jit-code-cache-padding",
- code_size,
- kProtNone,
- error_msg,
- use_ashmem));
- if (post_code_map == nullptr) {
- return nullptr;
- }
- DCHECK_EQ(code_map->Size(), code_size);
- DCHECK_EQ(post_code_map->Size(), post_code_size + code_sync_size);
-
- std::unique_ptr<MemMap> code_sync_map(SplitMemMap(post_code_map.get(),
- "jit-code-sync",
- post_code_size,
- kProtCode,
- error_msg,
- use_ashmem));
- if (code_sync_map == nullptr) {
- return nullptr;
- }
- DCHECK_EQ(post_code_map->Size(), post_code_size);
- DCHECK_EQ(code_sync_map->Size(), code_sync_size);
-
- std::unique_ptr<MemMap> writable_code_map;
- if (use_two_mappings) {
- // Allocate the R/W view.
- writable_code_map.reset(MemMap::MapFile(code_size,
- kProtData,
- MAP_SHARED,
- writable_code_fd.get(),
- /* start */ 0,
- /* low_4gb */ true,
- "jit-writable-code",
- &error_str));
- if (writable_code_map == nullptr) {
- std::ostringstream oss;
- oss << "Failed to create writable code cache: " << error_str << " size=" << code_size;
- *error_msg = oss.str();
- return nullptr;
- }
- }
+ DCHECK_EQ(code_map->Begin(), divider);
data_size = initial_capacity / 2;
code_size = initial_capacity - data_size;
DCHECK_EQ(code_size + data_size, initial_capacity);
- return new JitCodeCache(writable_code_map.release(),
- code_map.release(),
- data_map.release(),
- code_sync_map.release(),
- code_size,
- data_size,
- max_capacity,
- garbage_collect_code);
+ return new JitCodeCache(
+ code_map, data_map.release(), code_size, data_size, max_capacity, garbage_collect_code);
}
-JitCodeCache::JitCodeCache(MemMap* writable_code_map,
- MemMap* executable_code_map,
+JitCodeCache::JitCodeCache(MemMap* code_map,
MemMap* data_map,
- MemMap* code_sync_map,
size_t initial_code_capacity,
size_t initial_data_capacity,
size_t max_capacity,
@@ -285,10 +143,8 @@
: lock_("Jit code cache", kJitCodeCacheLock),
lock_cond_("Jit code cache condition variable", lock_),
collection_in_progress_(false),
+ code_map_(code_map),
data_map_(data_map),
- executable_code_map_(executable_code_map),
- writable_code_map_(writable_code_map),
- code_sync_map_(code_sync_map),
max_capacity_(max_capacity),
current_capacity_(initial_code_capacity + initial_data_capacity),
code_end_(initial_code_capacity),
@@ -308,8 +164,7 @@
inline_cache_cond_("Jit inline cache condition variable", lock_) {
DCHECK_GE(max_capacity, initial_code_capacity + initial_data_capacity);
- MemMap* writable_map = GetWritableMemMap();
- code_mspace_ = create_mspace_with_base(writable_map->Begin(), code_end_, false /*locked*/);
+ code_mspace_ = create_mspace_with_base(code_map_->Begin(), code_end_, false /*locked*/);
data_mspace_ = create_mspace_with_base(data_map_->Begin(), data_end_, false /*locked*/);
if (code_mspace_ == nullptr || data_mspace_ == nullptr) {
@@ -318,10 +173,7 @@
SetFootprintLimit(current_capacity_);
- if (writable_code_map_ != nullptr) {
- CHECKED_MPROTECT(writable_code_map_->Begin(), writable_code_map_->Size(), kProtReadOnly);
- }
- CHECKED_MPROTECT(executable_code_map_->Begin(), executable_code_map_->Size(), kProtCode);
+ CHECKED_MPROTECT(code_map_->Begin(), code_map_->Size(), kProtCode);
CHECKED_MPROTECT(data_map_->Begin(), data_map_->Size(), kProtData);
VLOG(jit) << "Created jit code cache: initial data size="
@@ -331,7 +183,7 @@
}
bool JitCodeCache::ContainsPc(const void* ptr) const {
- return executable_code_map_->Begin() <= ptr && ptr < executable_code_map_->End();
+ return code_map_->Begin() <= ptr && ptr < code_map_->End();
}
bool JitCodeCache::ContainsMethod(ArtMethod* method) {
@@ -344,96 +196,27 @@
return false;
}
-/* This method is only for CHECK/DCHECK that pointers are within to a region. */
-static bool IsAddressInMap(const void* addr,
- const MemMap* mem_map,
- const char* check_name) {
- if (addr == nullptr || mem_map->HasAddress(addr)) {
- return true;
- }
- LOG(ERROR) << "Is" << check_name << "Address " << addr
- << " not in [" << reinterpret_cast<void*>(mem_map->Begin())
- << ", " << reinterpret_cast<void*>(mem_map->Begin() + mem_map->Size()) << ")";
- return false;
-}
-
-bool JitCodeCache::IsDataAddress(const void* raw_addr) const {
- return IsAddressInMap(raw_addr, data_map_.get(), "Data");
-}
-
-bool JitCodeCache::IsExecutableAddress(const void* raw_addr) const {
- return IsAddressInMap(raw_addr, executable_code_map_.get(), "Executable");
-}
-
-bool JitCodeCache::IsWritableAddress(const void* raw_addr) const {
- return IsAddressInMap(raw_addr, GetWritableMemMap(), "Writable");
-}
-
-// Convert one address within the source map to the same offset within the destination map.
-static void* ConvertAddress(const void* source_address,
- const MemMap* source_map,
- const MemMap* destination_map) {
- DCHECK(source_map->HasAddress(source_address)) << source_address;
- ptrdiff_t offset = reinterpret_cast<const uint8_t*>(source_address) - source_map->Begin();
- uintptr_t address = reinterpret_cast<uintptr_t>(destination_map->Begin()) + offset;
- return reinterpret_cast<void*>(address);
-}
-
-template <typename T>
-T* JitCodeCache::ToExecutableAddress(T* writable_address) const {
- CHECK(IsWritableAddress(writable_address));
- if (writable_address == nullptr) {
- return nullptr;
- }
- void* executable_address = ConvertAddress(writable_address,
- GetWritableMemMap(),
- executable_code_map_.get());
- CHECK(IsExecutableAddress(executable_address));
- return reinterpret_cast<T*>(executable_address);
-}
-
-void* JitCodeCache::ToWritableAddress(const void* executable_address) const {
- CHECK(IsExecutableAddress(executable_address));
- if (executable_address == nullptr) {
- return nullptr;
- }
- void* writable_address = ConvertAddress(executable_address,
- executable_code_map_.get(),
- GetWritableMemMap());
- CHECK(IsWritableAddress(writable_address));
- return writable_address;
-}
-
class ScopedCodeCacheWrite : ScopedTrace {
public:
- explicit ScopedCodeCacheWrite(JitCodeCache* code_cache)
- : ScopedTrace("ScopedCodeCacheWrite") {
+ explicit ScopedCodeCacheWrite(MemMap* code_map, bool only_for_tlb_shootdown = false)
+ : ScopedTrace("ScopedCodeCacheWrite"),
+ code_map_(code_map),
+ only_for_tlb_shootdown_(only_for_tlb_shootdown) {
ScopedTrace trace("mprotect all");
- int prot_to_start_writing = kProtAll;
- if (code_cache->writable_code_map_ == nullptr) {
- // If there is only one mapping, use the executable mapping and toggle between rwx and rx.
- prot_to_start_writing = kProtAll;
- prot_to_stop_writing_ = kProtCode;
- } else {
- // If there are two mappings, use the writable mapping and toggle between rw and r.
- prot_to_start_writing = kProtData;
- prot_to_stop_writing_ = kProtReadOnly;
- }
- writable_map_ = code_cache->GetWritableMemMap();
- // If we're using ScopedCacheWrite only for TLB shootdown, we limit the scope of mprotect to
- // one page.
- size_ = writable_map_->Size();
- CHECKED_MPROTECT(writable_map_->Begin(), size_, prot_to_start_writing);
+ CHECKED_MPROTECT(
+ code_map_->Begin(), only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), kProtAll);
}
~ScopedCodeCacheWrite() {
ScopedTrace trace("mprotect code");
- CHECKED_MPROTECT(writable_map_->Begin(), size_, prot_to_stop_writing_);
+ CHECKED_MPROTECT(
+ code_map_->Begin(), only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), kProtCode);
}
-
private:
- int prot_to_stop_writing_;
- MemMap* writable_map_;
- size_t size_;
+ MemMap* const code_map_;
+
+ // If we're using ScopedCacheWrite only for TLB shootdown, we limit the scope of mprotect to
+ // one page.
+ const bool only_for_tlb_shootdown_;
DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite);
};
@@ -448,6 +231,7 @@
size_t fp_spill_mask,
const uint8_t* code,
size_t code_size,
+ size_t data_size,
bool osr,
Handle<mirror::ObjectArray<mirror::Object>> roots,
bool has_should_deoptimize_flag,
@@ -462,6 +246,7 @@
fp_spill_mask,
code,
code_size,
+ data_size,
osr,
roots,
has_should_deoptimize_flag,
@@ -479,6 +264,7 @@
fp_spill_mask,
code,
code_size,
+ data_size,
osr,
roots,
has_should_deoptimize_flag,
@@ -540,10 +326,8 @@
}
}
-uint8_t* JitCodeCache::GetRootTable(const void* code_ptr, uint32_t* number_of_roots) {
- CHECK(IsExecutableAddress(code_ptr));
+static uint8_t* GetRootTable(const void* code_ptr, uint32_t* number_of_roots = nullptr) {
OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
- // GetOptimizedCodeInfoPtr uses offsets relative to the EXECUTABLE address.
uint8_t* data = method_header->GetOptimizedCodeInfoPtr();
uint32_t roots = GetNumberOfRoots(data);
if (number_of_roots != nullptr) {
@@ -588,8 +372,6 @@
void JitCodeCache::SweepRootTables(IsMarkedVisitor* visitor) {
MutexLock mu(Thread::Current(), lock_);
for (const auto& entry : method_code_map_) {
- // GetRootTable takes an EXECUTABLE address.
- CHECK(IsExecutableAddress(entry.first));
uint32_t number_of_roots = 0;
uint8_t* roots_data = GetRootTable(entry.first, &number_of_roots);
GcRoot<mirror::Object>* roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data);
@@ -627,19 +409,17 @@
}
}
-void JitCodeCache::FreeCodeAndData(const void* code_ptr) {
- CHECK(IsExecutableAddress(code_ptr));
+void JitCodeCache::FreeCode(const void* code_ptr) {
+ uintptr_t allocation = FromCodeToAllocation(code_ptr);
// Notify native debugger that we are about to remove the code.
// It does nothing if we are not using native debugger.
DeleteJITCodeEntryForAddress(reinterpret_cast<uintptr_t>(code_ptr));
- // GetRootTable takes an EXECUTABLE address.
FreeData(GetRootTable(code_ptr));
- FreeRawCode(reinterpret_cast<uint8_t*>(FromCodeToAllocation(code_ptr)));
+ FreeCode(reinterpret_cast<uint8_t*>(allocation));
}
void JitCodeCache::FreeAllMethodHeaders(
const std::unordered_set<OatQuickMethodHeader*>& method_headers) {
- // method_headers are expected to be in the executable region.
{
MutexLock mu(Thread::Current(), *Locks::cha_lock_);
Runtime::Current()->GetClassLinker()->GetClassHierarchyAnalysis()
@@ -651,9 +431,9 @@
// so it's possible for the same method_header to start representing
// different compile code.
MutexLock mu(Thread::Current(), lock_);
- ScopedCodeCacheWrite scc(this);
+ ScopedCodeCacheWrite scc(code_map_.get());
for (const OatQuickMethodHeader* method_header : method_headers) {
- FreeCodeAndData(method_header->GetCode());
+ FreeCode(method_header->GetCode());
}
}
@@ -670,10 +450,9 @@
// with the classlinker_classes_lock_ held, and suspending ourselves could
// lead to a deadlock.
{
- ScopedCodeCacheWrite scc(this);
+ ScopedCodeCacheWrite scc(code_map_.get());
for (auto it = method_code_map_.begin(); it != method_code_map_.end();) {
if (alloc.ContainsUnsafe(it->second)) {
- CHECK(IsExecutableAddress(OatQuickMethodHeader::FromCodePointer(it->first)));
method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->first));
it = method_code_map_.erase(it);
} else {
@@ -765,129 +544,6 @@
method->SetCounter(std::min(jit_warmup_threshold - 1, 1));
}
-static void FlushInstructionPiplines(uint8_t* sync_page) {
- // After updating the JIT code cache we need to force all CPUs to
- // flush their instruction pipelines. In the absence of system call
- // to do this explicitly, we can achieve this indirectly by toggling
- // permissions on an executable page. This should send an IPI to
- // each core to update the TLB entry with the interrupt raised on
- // each core causing the instruction pipeline to be flushed.
- CHECKED_MPROTECT(sync_page, kPageSize, kProtAll);
- // Ensure the sync_page is present otherwise a TLB update may not be
- // necessary.
- sync_page[0] = 0;
- CHECKED_MPROTECT(sync_page, kPageSize, kProtCode);
-}
-
-#ifdef __aarch64__
-
-static void FlushJitCodeCacheRange(uint8_t* code_ptr,
- uint8_t* writable_ptr,
- size_t code_size) {
- // Cache maintenance instructions can cause permission faults when a
- // page is not present (e.g. swapped out or not backed). These
- // faults should be handled by the kernel, but a bug in some Linux
- // kernels may surface these permission faults to user-land which
- // does not currently deal with them (b/63885946). To work around
- // this, we read a value from each page to fault it in before
- // attempting to perform cache maintenance operations.
- //
- // For reference, this behavior is caused by this commit:
- // https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c
-
- // The cache-line size could be probed for from the CPU, but
- // assuming a safe lower bound is safe for CPUs that have different
- // cache-line sizes for big and little cores.
- static const uintptr_t kSafeCacheLineSize = 32;
-
- // Ensure stores are present in L1 data cache.
- __asm __volatile("dsb ish" ::: "memory");
-
- volatile uint8_t mutant;
-
- // Push dirty cache-lines out to the point of unification (PoU). The
- // point of unification is the first point in the cache/memory
- // hierarchy where the instruction cache and data cache have the
- // same view of memory. The PoU is where an instruction fetch will
- // fetch the new code generated by the JIT.
- //
- // See: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch11s04.html
- uintptr_t writable_addr = RoundDown(reinterpret_cast<uintptr_t>(writable_ptr),
- kSafeCacheLineSize);
- uintptr_t writable_end = RoundUp(reinterpret_cast<uintptr_t>(writable_ptr) + code_size,
- kSafeCacheLineSize);
- while (writable_addr < writable_end) {
- // Read from the cache-line to minimize the chance that a cache
- // maintenance instruction causes a fault (see kernel bug comment
- // above).
- mutant = *reinterpret_cast<const uint8_t*>(writable_addr);
-
- // Flush cache-line
- __asm volatile("dc cvau, %0" :: "r"(writable_addr) : "memory");
- writable_addr += kSafeCacheLineSize;
- }
-
- __asm __volatile("dsb ish" ::: "memory");
-
- uintptr_t code_addr = RoundDown(reinterpret_cast<uintptr_t>(code_ptr), kSafeCacheLineSize);
- const uintptr_t code_end = RoundUp(reinterpret_cast<uintptr_t>(code_ptr) + code_size,
- kSafeCacheLineSize);
- while (code_addr < code_end) {
- // Read from the cache-line to minimize the chance that a cache
- // maintenance instruction causes a fault (see kernel bug comment
- // above).
- mutant = *reinterpret_cast<const uint8_t*>(code_addr);
-
- // Invalidating the data cache line is only strictly necessary
- // when the JIT code cache has two mappings (the default). We know
- // this cache line is clean so this is just invalidating it (using
- // "dc ivac" would be preferable, but counts as a write and this
- // memory may not be mapped write permission).
- __asm volatile("dc cvau, %0" :: "r"(code_addr) : "memory");
-
- // Invalidate the instruction cache line to force instructions in
- // range to be re-fetched following update.
- __asm volatile("ic ivau, %0" :: "r"(code_addr) : "memory");
-
- code_addr += kSafeCacheLineSize;
- }
-
- // Wait for code cache invalidations to complete.
- __asm __volatile("dsb ish" ::: "memory");
-
- // Reset fetched instruction stream.
- __asm __volatile("isb");
-}
-
-#else // __aarch64
-
-static void FlushJitCodeCacheRange(uint8_t* code_ptr,
- uint8_t* writable_ptr,
- size_t code_size) {
- if (writable_ptr != code_ptr) {
- // When there are two mappings of the JIT code cache, RX and
- // RW, flush the RW version first as we've just dirtied the
- // cache lines with new code. Flushing the RX version first
- // can cause a permission fault as the those addresses are not
- // writable, but can appear dirty in the cache. There is a lot
- // of potential subtlety here depending on how the cache is
- // indexed and tagged.
- //
- // Flushing the RX version after the RW version is just
- // invalidating cachelines in the instruction cache. This is
- // necessary as the instruction cache will often have a
- // different set of cache lines present and because the JIT
- // code cache can start a new function at any boundary within
- // a cache-line.
- FlushDataCache(reinterpret_cast<char*>(writable_ptr),
- reinterpret_cast<char*>(writable_ptr + code_size));
- }
- FlushInstructionCache(reinterpret_cast<char*>(code_ptr),
- reinterpret_cast<char*>(code_ptr + code_size));
-}
-
-#endif // __aarch64
-
uint8_t* JitCodeCache::CommitCodeInternal(Thread* self,
ArtMethod* method,
uint8_t* stack_map,
@@ -898,6 +554,7 @@
size_t fp_spill_mask,
const uint8_t* code,
size_t code_size,
+ size_t data_size,
bool osr,
Handle<mirror::ObjectArray<mirror::Object>> roots,
bool has_should_deoptimize_flag,
@@ -917,37 +574,35 @@
MutexLock mu(self, lock_);
WaitForPotentialCollectionToComplete(self);
{
- ScopedCodeCacheWrite scc(this);
+ ScopedCodeCacheWrite scc(code_map_.get());
memory = AllocateCode(total_size);
if (memory == nullptr) {
return nullptr;
}
- uint8_t* writable_ptr = memory + header_size;
- code_ptr = ToExecutableAddress(writable_ptr);
+ code_ptr = memory + header_size;
- std::copy(code, code + code_size, writable_ptr);
- OatQuickMethodHeader* writable_method_header =
- OatQuickMethodHeader::FromCodePointer(writable_ptr);
- // We need to be able to write the OatQuickMethodHeader, so we use writable_method_header.
- // Otherwise, the offsets encoded in OatQuickMethodHeader are used relative to an executable
- // address, so we use code_ptr.
- new (writable_method_header) OatQuickMethodHeader(
+ std::copy(code, code + code_size, code_ptr);
+ method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
+ new (method_header) OatQuickMethodHeader(
code_ptr - stack_map,
code_ptr - method_info,
frame_size_in_bytes,
core_spill_mask,
fp_spill_mask,
code_size);
-
- FlushJitCodeCacheRange(code_ptr, writable_ptr, code_size);
- FlushInstructionPiplines(code_sync_map_->Begin());
-
+ // Flush caches before we remove write permission because some ARMv8 Qualcomm kernels may
+ // trigger a segfault if a page fault occurs when requesting a cache maintenance operation.
+ // This is a kernel bug that we need to work around until affected devices (e.g. Nexus 5X and
+ // 6P) stop being supported or their kernels are fixed.
+ //
+ // For reference, this behavior is caused by this commit:
+ // https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c
+ FlushInstructionCache(reinterpret_cast<char*>(code_ptr),
+ reinterpret_cast<char*>(code_ptr + code_size));
DCHECK(!Runtime::Current()->IsAotCompiler());
if (has_should_deoptimize_flag) {
- writable_method_header->SetHasShouldDeoptimizeFlag();
+ method_header->SetHasShouldDeoptimizeFlag();
}
- // All the pointers exported from the cache are executable addresses.
- method_header = ToExecutableAddress(writable_method_header);
}
number_of_compilations_++;
@@ -986,14 +641,16 @@
// but below we still make the compiled code valid for the method.
MutexLock mu(self, lock_);
// Fill the root table before updating the entry point.
- CHECK(IsDataAddress(roots_data));
DCHECK_EQ(FromStackMapToRoots(stack_map), roots_data);
DCHECK_LE(roots_data, stack_map);
FillRootTable(roots_data, roots);
-
- // Ensure the updates to the root table are visible with a store fence.
- QuasiAtomic::ThreadFenceSequentiallyConsistent();
-
+ {
+ // Flush data cache, as compiled code references literals in it.
+ // We also need a TLB shootdown to act as memory barrier across cores.
+ ScopedCodeCacheWrite ccw(code_map_.get(), /* only_for_tlb_shootdown */ true);
+ FlushDataCache(reinterpret_cast<char*>(roots_data),
+ reinterpret_cast<char*>(roots_data + data_size));
+ }
method_code_map_.Put(code_ptr, method);
if (osr) {
number_of_osr_compilations_++;
@@ -1041,11 +698,11 @@
bool in_cache = false;
{
- ScopedCodeCacheWrite ccw(this);
+ ScopedCodeCacheWrite ccw(code_map_.get());
for (auto code_iter = method_code_map_.begin(); code_iter != method_code_map_.end();) {
if (code_iter->second == method) {
if (release_memory) {
- FreeCodeAndData(code_iter->first);
+ FreeCode(code_iter->first);
}
code_iter = method_code_map_.erase(code_iter);
in_cache = true;
@@ -1099,10 +756,10 @@
profiling_infos_.erase(profile);
}
method->SetProfilingInfo(nullptr);
- ScopedCodeCacheWrite ccw(this);
+ ScopedCodeCacheWrite ccw(code_map_.get());
for (auto code_iter = method_code_map_.begin(); code_iter != method_code_map_.end();) {
if (code_iter->second == method) {
- FreeCodeAndData(code_iter->first);
+ FreeCode(code_iter->first);
code_iter = method_code_map_.erase(code_iter);
continue;
}
@@ -1168,7 +825,6 @@
uint8_t* stack_map_data,
uint8_t* roots_data) {
DCHECK_EQ(FromStackMapToRoots(stack_map_data), roots_data);
- CHECK(IsDataAddress(roots_data));
MutexLock mu(self, lock_);
FreeData(reinterpret_cast<uint8_t*>(roots_data));
}
@@ -1290,11 +946,11 @@
void JitCodeCache::SetFootprintLimit(size_t new_footprint) {
size_t per_space_footprint = new_footprint / 2;
- CHECK(IsAlignedParam(per_space_footprint, kPageSize));
+ DCHECK(IsAlignedParam(per_space_footprint, kPageSize));
DCHECK_EQ(per_space_footprint * 2, new_footprint);
mspace_set_footprint_limit(data_mspace_, per_space_footprint);
{
- ScopedCodeCacheWrite scc(this);
+ ScopedCodeCacheWrite scc(code_map_.get());
mspace_set_footprint_limit(code_mspace_, per_space_footprint);
}
}
@@ -1315,9 +971,7 @@
current_capacity_ = max_capacity_;
}
- if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
- LOG(INFO) << "Increasing code cache capacity to " << PrettySize(current_capacity_);
- }
+ VLOG(jit) << "Increasing code cache capacity to " << PrettySize(current_capacity_);
SetFootprintLimit(current_capacity_);
@@ -1372,8 +1026,8 @@
number_of_collections_++;
live_bitmap_.reset(CodeCacheBitmap::Create(
"code-cache-bitmap",
- reinterpret_cast<uintptr_t>(executable_code_map_->Begin()),
- reinterpret_cast<uintptr_t>(executable_code_map_->Begin() + current_capacity_ / 2)));
+ reinterpret_cast<uintptr_t>(code_map_->Begin()),
+ reinterpret_cast<uintptr_t>(code_map_->Begin() + current_capacity_ / 2)));
collection_in_progress_ = true;
}
}
@@ -1388,21 +1042,17 @@
do_full_collection = ShouldDoFullCollection();
}
- if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
- LOG(INFO) << "Do "
- << (do_full_collection ? "full" : "partial")
- << " code cache collection, code="
- << PrettySize(CodeCacheSize())
- << ", data=" << PrettySize(DataCacheSize());
- }
+ VLOG(jit) << "Do "
+ << (do_full_collection ? "full" : "partial")
+ << " code cache collection, code="
+ << PrettySize(CodeCacheSize())
+ << ", data=" << PrettySize(DataCacheSize());
DoCollection(self, /* collect_profiling_info */ do_full_collection);
- if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
- LOG(INFO) << "After code cache collection, code="
- << PrettySize(CodeCacheSize())
- << ", data=" << PrettySize(DataCacheSize());
- }
+ VLOG(jit) << "After code cache collection, code="
+ << PrettySize(CodeCacheSize())
+ << ", data=" << PrettySize(DataCacheSize());
{
MutexLock mu(self, lock_);
@@ -1449,16 +1099,14 @@
std::unordered_set<OatQuickMethodHeader*> method_headers;
{
MutexLock mu(self, lock_);
- ScopedCodeCacheWrite scc(this);
+ ScopedCodeCacheWrite scc(code_map_.get());
// Iterate over all compiled code and remove entries that are not marked.
for (auto it = method_code_map_.begin(); it != method_code_map_.end();) {
const void* code_ptr = it->first;
- CHECK(IsExecutableAddress(code_ptr));
uintptr_t allocation = FromCodeToAllocation(code_ptr);
if (GetLiveBitmap()->Test(allocation)) {
++it;
} else {
- CHECK(IsExecutableAddress(it->first));
method_headers.insert(OatQuickMethodHeader::FromCodePointer(it->first));
it = method_code_map_.erase(it);
}
@@ -1501,7 +1149,6 @@
for (const auto& it : method_code_map_) {
ArtMethod* method = it.second;
const void* code_ptr = it.first;
- CHECK(IsExecutableAddress(code_ptr));
const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
if (method_header->GetEntryPoint() == method->GetEntryPointFromQuickCompiledCode()) {
GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(code_ptr));
@@ -1527,7 +1174,6 @@
// Free all profiling infos of methods not compiled nor being compiled.
auto profiling_kept_end = std::remove_if(profiling_infos_.begin(), profiling_infos_.end(),
[this] (ProfilingInfo* info) NO_THREAD_SAFETY_ANALYSIS {
- CHECK(IsDataAddress(info));
const void* ptr = info->GetMethod()->GetEntryPointFromQuickCompiledCode();
// We have previously cleared the ProfilingInfo pointer in the ArtMethod in the hope
// that the compiled code would not get revived. As mutator threads run concurrently,
@@ -1588,7 +1234,6 @@
--it;
const void* code_ptr = it->first;
- CHECK(IsExecutableAddress(code_ptr));
OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
if (!method_header->Contains(pc)) {
return nullptr;
@@ -1671,7 +1316,6 @@
// store in the ArtMethod's ProfilingInfo pointer.
QuasiAtomic::ThreadFenceRelease();
- CHECK(IsDataAddress(info));
method->SetProfilingInfo(info);
profiling_infos_.push_back(info);
histogram_profiling_info_memory_use_.AddValue(profile_info_size);
@@ -1684,8 +1328,7 @@
if (code_mspace_ == mspace) {
size_t result = code_end_;
code_end_ += increment;
- MemMap* writable_map = GetWritableMemMap();
- return reinterpret_cast<void*>(result + writable_map->Begin());
+ return reinterpret_cast<void*>(result + code_map_->Begin());
} else {
DCHECK_EQ(data_mspace_, mspace);
size_t result = data_end_;
@@ -1837,7 +1480,6 @@
size_t JitCodeCache::GetMemorySizeOfCodePointer(const void* ptr) {
MutexLock mu(Thread::Current(), lock_);
- CHECK(IsExecutableAddress(ptr));
return mspace_usable_size(reinterpret_cast<const void*>(FromCodeToAllocation(ptr)));
}
@@ -1873,27 +1515,22 @@
size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment);
// Ensure the header ends up at expected instruction alignment.
DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(result + header_size), alignment);
- CHECK(IsWritableAddress(result));
used_memory_for_code_ += mspace_usable_size(result);
return result;
}
-void JitCodeCache::FreeRawCode(void* code) {
- CHECK(IsExecutableAddress(code));
- void* writable_code = ToWritableAddress(code);
- used_memory_for_code_ -= mspace_usable_size(writable_code);
- mspace_free(code_mspace_, writable_code);
+void JitCodeCache::FreeCode(uint8_t* code) {
+ used_memory_for_code_ -= mspace_usable_size(code);
+ mspace_free(code_mspace_, code);
}
uint8_t* JitCodeCache::AllocateData(size_t data_size) {
void* result = mspace_malloc(data_mspace_, data_size);
- CHECK(IsDataAddress(reinterpret_cast<uint8_t*>(result)));
used_memory_for_data_ += mspace_usable_size(result);
return reinterpret_cast<uint8_t*>(result);
}
void JitCodeCache::FreeData(uint8_t* data) {
- CHECK(IsDataAddress(data));
used_memory_for_data_ -= mspace_usable_size(data);
mspace_free(data_mspace_, data);
}
diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h
index 175501f..daa1d61 100644
--- a/runtime/jit/jit_code_cache.h
+++ b/runtime/jit/jit_code_cache.h
@@ -113,6 +113,7 @@
size_t fp_spill_mask,
const uint8_t* code,
size_t code_size,
+ size_t data_size,
bool osr,
Handle<mirror::ObjectArray<mirror::Object>> roots,
bool has_should_deoptimize_flag,
@@ -228,8 +229,6 @@
REQUIRES(!lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
- uint8_t* GetRootTable(const void* code_ptr, uint32_t* number_of_roots = nullptr);
-
// The GC needs to disallow the reading of inline caches when it processes them,
// to avoid having a class being used while it is being deleted.
void AllowInlineCacheAccess() REQUIRES(!lock_);
@@ -248,13 +247,9 @@
}
private:
- friend class ScopedCodeCacheWrite;
-
// Take ownership of maps.
JitCodeCache(MemMap* code_map,
MemMap* data_map,
- MemMap* writable_code_map,
- MemMap* code_sync_map,
size_t initial_code_capacity,
size_t initial_data_capacity,
size_t max_capacity,
@@ -272,6 +267,7 @@
size_t fp_spill_mask,
const uint8_t* code,
size_t code_size,
+ size_t data_size,
bool osr,
Handle<mirror::ObjectArray<mirror::Object>> roots,
bool has_should_deoptimize_flag,
@@ -296,7 +292,7 @@
REQUIRES(!Locks::cha_lock_);
// Free in the mspace allocations for `code_ptr`.
- void FreeCodeAndData(const void* code_ptr) REQUIRES(lock_);
+ void FreeCode(const void* code_ptr) REQUIRES(lock_);
// Number of bytes allocated in the code cache.
size_t CodeCacheSizeLocked() REQUIRES(lock_);
@@ -329,7 +325,7 @@
bool CheckLiveCompiledCodeHasProfilingInfo()
REQUIRES(lock_);
- void FreeRawCode(void* code) REQUIRES(lock_);
+ void FreeCode(uint8_t* code) REQUIRES(lock_);
uint8_t* AllocateCode(size_t code_size) REQUIRES(lock_);
void FreeData(uint8_t* data) REQUIRES(lock_);
uint8_t* AllocateData(size_t data_size) REQUIRES(lock_);
@@ -339,61 +335,25 @@
REQUIRES(!lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
- MemMap* GetWritableMemMap() const {
- if (writable_code_map_ == nullptr) {
- // The system required us to map the JIT Code Cache RWX (see
- // JitCodeCache::Create()).
- return executable_code_map_.get();
- } else {
- // Executable code is mapped RX, and writable code is mapped RW
- // to the underlying same memory, but at a different address.
- return writable_code_map_.get();
- }
- }
-
- bool IsDataAddress(const void* raw_addr) const;
-
- bool IsExecutableAddress(const void* raw_addr) const;
-
- bool IsWritableAddress(const void* raw_addr) const;
-
- template <typename T>
- T* ToExecutableAddress(T* writable_address) const;
-
- void* ToWritableAddress(const void* executable_address) const;
-
// Lock for guarding allocations, collections, and the method_code_map_.
Mutex lock_;
// Condition to wait on during collection.
ConditionVariable lock_cond_ GUARDED_BY(lock_);
// Whether there is a code cache collection in progress.
bool collection_in_progress_ GUARDED_BY(lock_);
- // JITting methods obviously requires both write and execute permissions on a region of memory.
- // In tye typical (non-debugging) case, we separate the memory mapped view that can write the code
- // from a view that the runtime uses to execute the code. Having these two views eliminates any
- // single address region having rwx permissions. An attacker could still write the writable
- // address and then execute the executable address. We allocate the mappings with a random
- // address relationship to each other which makes the attacker need two addresses rather than
- // just one. In the debugging case there is no file descriptor to back the
- // shared memory, and hence we have to use a single mapping.
+ // Mem map which holds code.
+ std::unique_ptr<MemMap> code_map_;
// Mem map which holds data (stack maps and profiling info).
std::unique_ptr<MemMap> data_map_;
- // Mem map which holds a non-writable view of code for JIT.
- std::unique_ptr<MemMap> executable_code_map_;
- // Mem map which holds a non-executable view of code for JIT.
- std::unique_ptr<MemMap> writable_code_map_;
- // Mem map which holds one executable page that we use for flushing instruction
- // fetch buffers. The code on this page is never executed.
- std::unique_ptr<MemMap> code_sync_map_;
// The opaque mspace for allocating code.
void* code_mspace_ GUARDED_BY(lock_);
// The opaque mspace for allocating data.
void* data_mspace_ GUARDED_BY(lock_);
// Bitmap for collecting code and data.
std::unique_ptr<CodeCacheBitmap> live_bitmap_;
- // Holds non-writable compiled code associated to the ArtMethod.
+ // Holds compiled code associated to the ArtMethod.
SafeMap<const void*, ArtMethod*> method_code_map_ GUARDED_BY(lock_);
- // Holds non-writable osr compiled code associated to the ArtMethod.
+ // Holds osr compiled code associated to the ArtMethod.
SafeMap<ArtMethod*, const void*> osr_code_map_ GUARDED_BY(lock_);
// ProfilingInfo objects we have allocated.
std::vector<ProfilingInfo*> profiling_infos_ GUARDED_BY(lock_);
diff --git a/runtime/mem_map.cc b/runtime/mem_map.cc
index 4e82480..743604c 100644
--- a/runtime/mem_map.cc
+++ b/runtime/mem_map.cc
@@ -496,7 +496,7 @@
MEMORY_TOOL_MAKE_UNDEFINED(base_begin_, base_size_);
int result = munmap(base_begin_, base_size_);
if (result == -1) {
- PLOG(FATAL) << "munmap failed: " << BaseBegin() << "..." << BaseEnd();
+ PLOG(FATAL) << "munmap failed";
}
}
@@ -535,13 +535,8 @@
}
}
-MemMap* MemMap::RemapAtEnd(uint8_t* new_end,
- const char* tail_name,
- int tail_prot,
- int sharing_flags,
- std::string* error_msg,
- bool use_ashmem,
- unique_fd* shmem_fd) {
+MemMap* MemMap::RemapAtEnd(uint8_t* new_end, const char* tail_name, int tail_prot,
+ std::string* error_msg, bool use_ashmem) {
use_ashmem = use_ashmem && !kIsTargetLinux;
DCHECK_GE(new_end, Begin());
DCHECK_LE(new_end, End());
@@ -560,12 +555,6 @@
size_ = new_end - reinterpret_cast<uint8_t*>(begin_);
base_size_ = new_base_end - reinterpret_cast<uint8_t*>(base_begin_);
DCHECK_LE(begin_ + size_, reinterpret_cast<uint8_t*>(base_begin_) + base_size_);
- if (base_size_ == 0u) {
- // All pages in this MemMap have been handed out. Invalidate base
- // pointer to prevent the destructor calling munmap() on
- // zero-length region (which can't succeed).
- base_begin_ = nullptr;
- }
size_t tail_size = old_end - new_end;
uint8_t* tail_base_begin = new_base_end;
size_t tail_base_size = old_base_end - new_base_end;
@@ -573,14 +562,14 @@
DCHECK_ALIGNED(tail_base_size, kPageSize);
unique_fd fd;
- int flags = MAP_ANONYMOUS | sharing_flags;
+ int flags = MAP_PRIVATE | MAP_ANONYMOUS;
if (use_ashmem) {
// android_os_Debug.cpp read_mapinfo assumes all ashmem regions associated with the VM are
// prefixed "dalvik-".
std::string debug_friendly_name("dalvik-");
debug_friendly_name += tail_name;
fd.reset(ashmem_create_region(debug_friendly_name.c_str(), tail_base_size));
- flags = MAP_FIXED | sharing_flags;
+ flags = MAP_PRIVATE | MAP_FIXED;
if (fd.get() == -1) {
*error_msg = StringPrintf("ashmem_create_region failed for '%s': %s",
tail_name, strerror(errno));
@@ -614,9 +603,6 @@
fd.get());
return nullptr;
}
- if (shmem_fd != nullptr) {
- shmem_fd->reset(fd.release());
- }
return new MemMap(tail_name, actual, tail_size, actual, tail_base_size, tail_prot, false);
}
diff --git a/runtime/mem_map.h b/runtime/mem_map.h
index d8908ad..5603963 100644
--- a/runtime/mem_map.h
+++ b/runtime/mem_map.h
@@ -25,7 +25,6 @@
#include <string>
#include "android-base/thread_annotations.h"
-#include "android-base/unique_fd.h"
namespace art {
@@ -38,8 +37,6 @@
#define USE_ART_LOW_4G_ALLOCATOR 0
#endif
-using android::base::unique_fd;
-
#ifdef __linux__
static constexpr bool kMadviseZeroes = true;
#else
@@ -171,14 +168,11 @@
}
// Unmap the pages at end and remap them to create another memory map.
- // sharing_flags should be either MAP_PRIVATE or MAP_SHARED.
MemMap* RemapAtEnd(uint8_t* new_end,
const char* tail_name,
int tail_prot,
- int sharing_flags,
std::string* error_msg,
- bool use_ashmem = true,
- unique_fd* shmem_fd = nullptr);
+ bool use_ashmem = true);
static bool CheckNoGaps(MemMap* begin_map, MemMap* end_map)
REQUIRES(!MemMap::mem_maps_lock_);
diff --git a/runtime/mem_map_test.cc b/runtime/mem_map_test.cc
index 99bf004..a4ebb16 100644
--- a/runtime/mem_map_test.cc
+++ b/runtime/mem_map_test.cc
@@ -74,7 +74,6 @@
MemMap* m1 = m0->RemapAtEnd(base0 + page_size,
"MemMapTest_RemapAtEndTest_map1",
PROT_READ | PROT_WRITE,
- MAP_PRIVATE,
&error_msg);
// Check the states of the two maps.
EXPECT_EQ(m0->Begin(), base0) << error_msg;
@@ -457,7 +456,6 @@
std::unique_ptr<MemMap> m1(m0->RemapAtEnd(base0 + 3 * page_size,
"MemMapTest_AlignByTest_map1",
PROT_READ | PROT_WRITE,
- MAP_PRIVATE,
&error_msg));
uint8_t* base1 = m1->Begin();
ASSERT_TRUE(base1 != nullptr) << error_msg;
@@ -467,7 +465,6 @@
std::unique_ptr<MemMap> m2(m1->RemapAtEnd(base1 + 4 * page_size,
"MemMapTest_AlignByTest_map2",
PROT_READ | PROT_WRITE,
- MAP_PRIVATE,
&error_msg));
uint8_t* base2 = m2->Begin();
ASSERT_TRUE(base2 != nullptr) << error_msg;
@@ -477,7 +474,6 @@
std::unique_ptr<MemMap> m3(m2->RemapAtEnd(base2 + 3 * page_size,
"MemMapTest_AlignByTest_map1",
PROT_READ | PROT_WRITE,
- MAP_PRIVATE,
&error_msg));
uint8_t* base3 = m3->Begin();
ASSERT_TRUE(base3 != nullptr) << error_msg;
diff --git a/runtime/mirror/accessible_object.h b/runtime/mirror/accessible_object.h
index a217193..d489f14 100644
--- a/runtime/mirror/accessible_object.h
+++ b/runtime/mirror/accessible_object.h
@@ -17,11 +17,8 @@
#ifndef ART_RUNTIME_MIRROR_ACCESSIBLE_OBJECT_H_
#define ART_RUNTIME_MIRROR_ACCESSIBLE_OBJECT_H_
-#include "class.h"
-#include "gc_root.h"
#include "object.h"
#include "read_barrier_option.h"
-#include "thread.h"
namespace art {
@@ -34,12 +31,6 @@
return OFFSET_OF_OBJECT_MEMBER(AccessibleObject, flag_);
}
- template<bool kTransactionActive>
- void SetAccessible(bool value) REQUIRES_SHARED(Locks::mutator_lock_) {
- UNUSED(padding_);
- return SetFieldBoolean<kTransactionActive>(FlagOffset(), value ? 1u : 0u);
- }
-
bool IsAccessible() REQUIRES_SHARED(Locks::mutator_lock_) {
return GetFieldBoolean(FlagOffset());
}
@@ -47,7 +38,7 @@
private:
uint8_t flag_;
// Padding required for correct alignment of subclasses like Executable, Field, etc.
- uint8_t padding_[1];
+ uint8_t padding_[1] ATTRIBUTE_UNUSED;
DISALLOW_IMPLICIT_CONSTRUCTORS(AccessibleObject);
};
diff --git a/runtime/mirror/array-inl.h b/runtime/mirror/array-inl.h
index 63142d5..2281245 100644
--- a/runtime/mirror/array-inl.h
+++ b/runtime/mirror/array-inl.h
@@ -27,8 +27,7 @@
#include "class.h"
#include "gc/heap-inl.h"
#include "obj_ptr-inl.h"
-#include "object-inl.h"
-#include "thread.h"
+#include "thread-current-inl.h"
namespace art {
namespace mirror {
diff --git a/runtime/mirror/field-inl.h b/runtime/mirror/field-inl.h
index d33df5c..ad48202 100644
--- a/runtime/mirror/field-inl.h
+++ b/runtime/mirror/field-inl.h
@@ -20,7 +20,8 @@
#include "field.h"
#include "art_field-inl.h"
-#include "mirror/dex_cache-inl.h"
+#include "class-inl.h"
+#include "dex_cache-inl.h"
namespace art {
@@ -87,6 +88,10 @@
SetFieldObject<kTransactionActive>(OFFSET_OF_OBJECT_MEMBER(Field, type_), type);
}
+inline Primitive::Type Field::GetTypeAsPrimitiveType() {
+ return GetType()->GetPrimitiveType();
+}
+
} // namespace mirror
} // namespace art
diff --git a/runtime/mirror/field.h b/runtime/mirror/field.h
index 40186a6..6845575 100644
--- a/runtime/mirror/field.h
+++ b/runtime/mirror/field.h
@@ -20,8 +20,10 @@
#include "accessible_object.h"
#include "base/enums.h"
#include "gc_root.h"
+#include "modifiers.h"
#include "obj_ptr.h"
#include "object.h"
+#include "primitive.h"
#include "read_barrier_option.h"
namespace art {
@@ -69,10 +71,7 @@
return (GetAccessFlags() & kAccVolatile) != 0;
}
- ALWAYS_INLINE Primitive::Type GetTypeAsPrimitiveType()
- REQUIRES_SHARED(Locks::mutator_lock_) {
- return GetType()->GetPrimitiveType();
- }
+ ALWAYS_INLINE Primitive::Type GetTypeAsPrimitiveType() REQUIRES_SHARED(Locks::mutator_lock_);
mirror::Class* GetType() REQUIRES_SHARED(Locks::mutator_lock_) {
return GetFieldObject<mirror::Class>(OFFSET_OF_OBJECT_MEMBER(Field, type_));
diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h
index 086925b..6eb200d 100644
--- a/runtime/mirror/object-inl.h
+++ b/runtime/mirror/object-inl.h
@@ -393,14 +393,6 @@
}
template<VerifyObjectFlags kVerifyFlags, bool kIsVolatile>
-inline uint8_t Object::GetFieldBoolean(MemberOffset field_offset) {
- if (kVerifyFlags & kVerifyThis) {
- VerifyObject(this);
- }
- return GetField<uint8_t, kIsVolatile>(field_offset);
-}
-
-template<VerifyObjectFlags kVerifyFlags, bool kIsVolatile>
inline int8_t Object::GetFieldByte(MemberOffset field_offset) {
if (kVerifyFlags & kVerifyThis) {
VerifyObject(this);
@@ -724,11 +716,10 @@
}
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
HeapReference<T>* objref_addr = reinterpret_cast<HeapReference<T>*>(raw_addr);
- T* result = ReadBarrier::Barrier<T, kReadBarrierOption>(this, field_offset, objref_addr);
- if (kIsVolatile) {
- // TODO: Refactor to use a SequentiallyConsistent load instead.
- QuasiAtomic::ThreadFenceAcquire(); // Ensure visibility of operations preceding store.
- }
+ T* result = ReadBarrier::Barrier<T, kIsVolatile, kReadBarrierOption>(
+ this,
+ field_offset,
+ objref_addr);
if (kVerifyFlags & kVerifyReads) {
VerifyObject(result);
}
@@ -764,15 +755,7 @@
}
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
HeapReference<Object>* objref_addr = reinterpret_cast<HeapReference<Object>*>(raw_addr);
- if (kIsVolatile) {
- // TODO: Refactor to use a SequentiallyConsistent store instead.
- QuasiAtomic::ThreadFenceRelease(); // Ensure that prior accesses are visible before store.
- objref_addr->Assign(new_value.Ptr());
- QuasiAtomic::ThreadFenceSequentiallyConsistent();
- // Ensure this store occurs before any volatile loads.
- } else {
- objref_addr->Assign(new_value.Ptr());
- }
+ objref_addr->Assign<kIsVolatile>(new_value.Ptr());
}
template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags,
@@ -843,13 +826,12 @@
if (kTransactionActive) {
Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
}
- HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
- HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
+ uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
+ uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
- bool success = atomic_addr->CompareExchangeWeakSequentiallyConsistent(old_ref.reference_,
- new_ref.reference_);
+ bool success = atomic_addr->CompareExchangeWeakSequentiallyConsistent(old_ref, new_ref);
return success;
}
@@ -885,13 +867,12 @@
if (kTransactionActive) {
Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
}
- HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
- HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
+ uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
+ uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
- bool success = atomic_addr->CompareExchangeStrongSequentiallyConsistent(old_ref.reference_,
- new_ref.reference_);
+ bool success = atomic_addr->CompareExchangeStrongSequentiallyConsistent(old_ref, new_ref);
return success;
}
@@ -915,13 +896,12 @@
if (kTransactionActive) {
Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
}
- HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
- HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
+ uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
+ uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
- bool success = atomic_addr->CompareExchangeWeakRelaxed(old_ref.reference_,
- new_ref.reference_);
+ bool success = atomic_addr->CompareExchangeWeakRelaxed(old_ref, new_ref);
return success;
}
@@ -945,13 +925,12 @@
if (kTransactionActive) {
Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
}
- HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
- HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
+ uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
+ uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
- bool success = atomic_addr->CompareExchangeWeakRelease(old_ref.reference_,
- new_ref.reference_);
+ bool success = atomic_addr->CompareExchangeWeakRelease(old_ref, new_ref);
return success;
}
diff --git a/runtime/mirror/object-readbarrier-inl.h b/runtime/mirror/object-readbarrier-inl.h
index 69365af..f076940 100644
--- a/runtime/mirror/object-readbarrier-inl.h
+++ b/runtime/mirror/object-readbarrier-inl.h
@@ -211,13 +211,12 @@
if (kTransactionActive) {
Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
}
- HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
- HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
+ uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
+ uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
- bool success = atomic_addr->CompareExchangeStrongRelaxed(old_ref.reference_,
- new_ref.reference_);
+ bool success = atomic_addr->CompareExchangeStrongRelaxed(old_ref, new_ref);
return success;
}
@@ -241,13 +240,12 @@
if (kTransactionActive) {
Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
}
- HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
- HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
+ uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
+ uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
- bool success = atomic_addr->CompareExchangeStrongRelease(old_ref.reference_,
- new_ref.reference_);
+ bool success = atomic_addr->CompareExchangeStrongRelease(old_ref, new_ref);
return success;
}
diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h
index 886780f..aedcd66 100644
--- a/runtime/mirror/object.h
+++ b/runtime/mirror/object.h
@@ -380,7 +380,12 @@
template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, bool kIsVolatile = false>
ALWAYS_INLINE uint8_t GetFieldBoolean(MemberOffset field_offset)
- REQUIRES_SHARED(Locks::mutator_lock_);
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ if (kVerifyFlags & kVerifyThis) {
+ VerifyObject(this);
+ }
+ return GetField<uint8_t, kIsVolatile>(field_offset);
+ }
template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, bool kIsVolatile = false>
ALWAYS_INLINE int8_t GetFieldByte(MemberOffset field_offset)
diff --git a/runtime/mirror/object_reference-inl.h b/runtime/mirror/object_reference-inl.h
index 22fb83c..60f3ce1 100644
--- a/runtime/mirror/object_reference-inl.h
+++ b/runtime/mirror/object_reference-inl.h
@@ -30,17 +30,10 @@
}
template<class MirrorType>
-HeapReference<MirrorType> HeapReference<MirrorType>::FromObjPtr(ObjPtr<MirrorType> ptr) {
- return HeapReference<MirrorType>(ptr.Ptr());
-}
-
-template<class MirrorType>
bool HeapReference<MirrorType>::CasWeakRelaxed(MirrorType* expected_ptr, MirrorType* new_ptr) {
- HeapReference<Object> expected_ref(HeapReference<Object>::FromMirrorPtr(expected_ptr));
- HeapReference<Object> new_ref(HeapReference<Object>::FromMirrorPtr(new_ptr));
- Atomic<uint32_t>* atomic_reference = reinterpret_cast<Atomic<uint32_t>*>(&this->reference_);
- return atomic_reference->CompareExchangeWeakRelaxed(expected_ref.reference_,
- new_ref.reference_);
+ return reference_.CompareExchangeWeakRelaxed(
+ Compression::Compress(expected_ptr),
+ Compression::Compress(new_ptr));
}
} // namespace mirror
diff --git a/runtime/mirror/object_reference.h b/runtime/mirror/object_reference.h
index a96a120..c62ee6c 100644
--- a/runtime/mirror/object_reference.h
+++ b/runtime/mirror/object_reference.h
@@ -17,6 +17,7 @@
#ifndef ART_RUNTIME_MIRROR_OBJECT_REFERENCE_H_
#define ART_RUNTIME_MIRROR_OBJECT_REFERENCE_H_
+#include "atomic.h"
#include "base/mutex.h" // For Locks::mutator_lock_.
#include "globals.h"
#include "obj_ptr.h"
@@ -30,20 +31,43 @@
// extra platform specific padding.
#define MANAGED PACKED(4)
+template<bool kPoisonReferences, class MirrorType>
+class PtrCompression {
+ public:
+ // Compress reference to its bit representation.
+ static uint32_t Compress(MirrorType* mirror_ptr) {
+ uintptr_t as_bits = reinterpret_cast<uintptr_t>(mirror_ptr);
+ return static_cast<uint32_t>(kPoisonReferences ? -as_bits : as_bits);
+ }
+
+ // Uncompress an encoded reference from its bit representation.
+ static MirrorType* Decompress(uint32_t ref) {
+ uintptr_t as_bits = kPoisonReferences ? -ref : ref;
+ return reinterpret_cast<MirrorType*>(as_bits);
+ }
+
+ // Convert an ObjPtr to a compressed reference.
+ static uint32_t Compress(ObjPtr<MirrorType> ptr) REQUIRES_SHARED(Locks::mutator_lock_) {
+ return Compress(ptr.Ptr());
+ }
+};
+
// Value type representing a reference to a mirror::Object of type MirrorType.
template<bool kPoisonReferences, class MirrorType>
class MANAGED ObjectReference {
+ private:
+ using Compression = PtrCompression<kPoisonReferences, MirrorType>;
+
public:
- MirrorType* AsMirrorPtr() const REQUIRES_SHARED(Locks::mutator_lock_) {
- return UnCompress();
+ MirrorType* AsMirrorPtr() const {
+ return Compression::Decompress(reference_);
}
- void Assign(MirrorType* other) REQUIRES_SHARED(Locks::mutator_lock_) {
- reference_ = Compress(other);
+ void Assign(MirrorType* other) {
+ reference_ = Compression::Compress(other);
}
- void Assign(ObjPtr<MirrorType> ptr)
- REQUIRES_SHARED(Locks::mutator_lock_);
+ void Assign(ObjPtr<MirrorType> ptr) REQUIRES_SHARED(Locks::mutator_lock_);
void Clear() {
reference_ = 0;
@@ -58,48 +82,71 @@
return reference_;
}
+ static ObjectReference<kPoisonReferences, MirrorType> FromMirrorPtr(MirrorType* mirror_ptr)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ return ObjectReference<kPoisonReferences, MirrorType>(mirror_ptr);
+ }
+
protected:
- explicit ObjectReference(MirrorType* mirror_ptr)
- REQUIRES_SHARED(Locks::mutator_lock_)
- : reference_(Compress(mirror_ptr)) {
+ explicit ObjectReference(MirrorType* mirror_ptr) REQUIRES_SHARED(Locks::mutator_lock_)
+ : reference_(Compression::Compress(mirror_ptr)) {
}
- // Compress reference to its bit representation.
- static uint32_t Compress(MirrorType* mirror_ptr) REQUIRES_SHARED(Locks::mutator_lock_) {
- uintptr_t as_bits = reinterpret_cast<uintptr_t>(mirror_ptr);
- return static_cast<uint32_t>(kPoisonReferences ? -as_bits : as_bits);
- }
-
- // Uncompress an encoded reference from its bit representation.
- MirrorType* UnCompress() const REQUIRES_SHARED(Locks::mutator_lock_) {
- uintptr_t as_bits = kPoisonReferences ? -reference_ : reference_;
- return reinterpret_cast<MirrorType*>(as_bits);
- }
-
- friend class Object;
-
// The encoded reference to a mirror::Object.
uint32_t reference_;
};
// References between objects within the managed heap.
+// Similar API to ObjectReference, but not a value type. Supports atomic access.
template<class MirrorType>
-class MANAGED HeapReference : public ObjectReference<kPoisonHeapReferences, MirrorType> {
+class MANAGED HeapReference {
+ private:
+ using Compression = PtrCompression<kPoisonHeapReferences, MirrorType>;
+
public:
+ HeapReference() REQUIRES_SHARED(Locks::mutator_lock_) : HeapReference(nullptr) {}
+
+ template <bool kIsVolatile = false>
+ MirrorType* AsMirrorPtr() const REQUIRES_SHARED(Locks::mutator_lock_) {
+ return Compression::Decompress(
+ kIsVolatile ? reference_.LoadSequentiallyConsistent() : reference_.LoadJavaData());
+ }
+
+ template <bool kIsVolatile = false>
+ void Assign(MirrorType* other) REQUIRES_SHARED(Locks::mutator_lock_) {
+ if (kIsVolatile) {
+ reference_.StoreSequentiallyConsistent(Compression::Compress(other));
+ } else {
+ reference_.StoreJavaData(Compression::Compress(other));
+ }
+ }
+
+ template <bool kIsVolatile = false>
+ void Assign(ObjPtr<MirrorType> ptr) REQUIRES_SHARED(Locks::mutator_lock_);
+
+ void Clear() {
+ reference_.StoreJavaData(0);
+ DCHECK(IsNull());
+ }
+
+ bool IsNull() const {
+ return reference_.LoadJavaData() == 0;
+ }
+
static HeapReference<MirrorType> FromMirrorPtr(MirrorType* mirror_ptr)
REQUIRES_SHARED(Locks::mutator_lock_) {
return HeapReference<MirrorType>(mirror_ptr);
}
- static HeapReference<MirrorType> FromObjPtr(ObjPtr<MirrorType> ptr)
- REQUIRES_SHARED(Locks::mutator_lock_);
-
bool CasWeakRelaxed(MirrorType* old_ptr, MirrorType* new_ptr)
REQUIRES_SHARED(Locks::mutator_lock_);
private:
explicit HeapReference(MirrorType* mirror_ptr) REQUIRES_SHARED(Locks::mutator_lock_)
- : ObjectReference<kPoisonHeapReferences, MirrorType>(mirror_ptr) {}
+ : reference_(Compression::Compress(mirror_ptr)) {}
+
+ // The encoded reference to a mirror::Object. Atomically updateable.
+ Atomic<uint32_t> reference_;
};
static_assert(sizeof(mirror::HeapReference<mirror::Object>) == kHeapReferenceSize,
diff --git a/runtime/native/java_lang_reflect_Field.cc b/runtime/native/java_lang_reflect_Field.cc
index 9f59a1f..2e4dd8a 100644
--- a/runtime/native/java_lang_reflect_Field.cc
+++ b/runtime/native/java_lang_reflect_Field.cc
@@ -27,7 +27,7 @@
#include "dex_file_annotations.h"
#include "jni_internal.h"
#include "mirror/class-inl.h"
-#include "mirror/field.h"
+#include "mirror/field-inl.h"
#include "native_util.h"
#include "reflection-inl.h"
#include "scoped_fast_native_object_access-inl.h"
diff --git a/runtime/native/sun_misc_Unsafe.cc b/runtime/native/sun_misc_Unsafe.cc
index 761362f..b2bdeed 100644
--- a/runtime/native/sun_misc_Unsafe.cc
+++ b/runtime/native/sun_misc_Unsafe.cc
@@ -67,10 +67,12 @@
if (kUseReadBarrier) {
// Need to make sure the reference stored in the field is a to-space one before attempting the
// CAS or the CAS could fail incorrectly.
+ // Note that the read barrier load does NOT need to be volatile.
mirror::HeapReference<mirror::Object>* field_addr =
reinterpret_cast<mirror::HeapReference<mirror::Object>*>(
reinterpret_cast<uint8_t*>(obj.Ptr()) + static_cast<size_t>(offset));
- ReadBarrier::Barrier<mirror::Object, kWithReadBarrier, /* kAlwaysUpdateField */ true>(
+ ReadBarrier::Barrier<mirror::Object, /* kIsVolatile */ false, kWithReadBarrier,
+ /* kAlwaysUpdateField */ true>(
obj.Ptr(),
MemberOffset(offset),
field_addr);
@@ -112,6 +114,7 @@
jint newValue) {
ScopedFastNativeObjectAccess soa(env);
ObjPtr<mirror::Object> obj = soa.Decode<mirror::Object>(javaObj);
+ // TODO: A release store is likely to be faster on future processors.
QuasiAtomic::ThreadFenceRelease();
// JNI must use non transactional mode.
obj->SetField32<false>(MemberOffset(offset), newValue);
diff --git a/runtime/quick_exception_handler.cc b/runtime/quick_exception_handler.cc
index db10103..b592247 100644
--- a/runtime/quick_exception_handler.cc
+++ b/runtime/quick_exception_handler.cc
@@ -533,21 +533,19 @@
void QuickExceptionHandler::DeoptimizeSingleFrame(DeoptimizationKind kind) {
DCHECK(is_deoptimization_);
- if (VLOG_IS_ON(deopt) || kDebugExceptionDelivery) {
- LOG(INFO) << "Single-frame deopting:";
- DumpFramesWithType(self_, true);
- }
-
DeoptimizeStackVisitor visitor(self_, context_, this, true);
visitor.WalkStack(true);
// Compiled code made an explicit deoptimization.
ArtMethod* deopt_method = visitor.GetSingleFrameDeoptMethod();
DCHECK(deopt_method != nullptr);
- LOG(INFO) << "Deoptimizing "
- << deopt_method->PrettyMethod()
- << " due to "
- << GetDeoptimizationKindName(kind);
+ if (VLOG_IS_ON(deopt) || kDebugExceptionDelivery) {
+ LOG(INFO) << "Single-frame deopting: "
+ << deopt_method->PrettyMethod()
+ << " due to "
+ << GetDeoptimizationKindName(kind);
+ DumpFramesWithType(self_, /* details */ true);
+ }
if (Runtime::Current()->UseJitCompilation()) {
Runtime::Current()->GetJit()->GetCodeCache()->InvalidateCompiledCodeFor(
deopt_method, visitor.GetSingleFrameDeoptQuickMethodHeader());
diff --git a/runtime/read_barrier-inl.h b/runtime/read_barrier-inl.h
index b0935c0..6424599 100644
--- a/runtime/read_barrier-inl.h
+++ b/runtime/read_barrier-inl.h
@@ -33,7 +33,8 @@
// Disabled for performance reasons.
static constexpr bool kCheckDebugDisallowReadBarrierCount = false;
-template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kAlwaysUpdateField>
+template <typename MirrorType, bool kIsVolatile, ReadBarrierOption kReadBarrierOption,
+ bool kAlwaysUpdateField>
inline MirrorType* ReadBarrier::Barrier(
mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr) {
constexpr bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
@@ -55,7 +56,7 @@
}
ref_addr = reinterpret_cast<mirror::HeapReference<MirrorType>*>(
fake_address_dependency | reinterpret_cast<uintptr_t>(ref_addr));
- MirrorType* ref = ref_addr->AsMirrorPtr();
+ MirrorType* ref = ref_addr->template AsMirrorPtr<kIsVolatile>();
MirrorType* old_ref = ref;
if (is_gray) {
// Slow-path.
@@ -71,9 +72,9 @@
return ref;
} else if (kUseBrooksReadBarrier) {
// To be implemented.
- return ref_addr->AsMirrorPtr();
+ return ref_addr->template AsMirrorPtr<kIsVolatile>();
} else if (kUseTableLookupReadBarrier) {
- MirrorType* ref = ref_addr->AsMirrorPtr();
+ MirrorType* ref = ref_addr->template AsMirrorPtr<kIsVolatile>();
MirrorType* old_ref = ref;
// The heap or the collector can be null at startup. TODO: avoid the need for this null check.
gc::Heap* heap = Runtime::Current()->GetHeap();
@@ -93,7 +94,7 @@
}
} else {
// No read barrier.
- return ref_addr->AsMirrorPtr();
+ return ref_addr->template AsMirrorPtr<kIsVolatile>();
}
}
diff --git a/runtime/read_barrier.h b/runtime/read_barrier.h
index d36acbc..8a106aa 100644
--- a/runtime/read_barrier.h
+++ b/runtime/read_barrier.h
@@ -46,9 +46,13 @@
// fast-debug environment.
DECLARE_RUNTIME_DEBUG_FLAG(kEnableReadBarrierInvariantChecks);
+ // Return the reference at ref_addr, invoking read barrier as appropriate.
+ // Ref_addr is an address within obj.
// It's up to the implementation whether the given field gets updated whereas the return value
// must be an updated reference unless kAlwaysUpdateField is true.
- template <typename MirrorType, ReadBarrierOption kReadBarrierOption = kWithReadBarrier,
+ template <typename MirrorType,
+ bool kIsVolatile,
+ ReadBarrierOption kReadBarrierOption = kWithReadBarrier,
bool kAlwaysUpdateField = false>
ALWAYS_INLINE static MirrorType* Barrier(
mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr)
diff --git a/runtime/runtime-inl.h b/runtime/runtime-inl.h
index 609f0d6..4584351 100644
--- a/runtime/runtime-inl.h
+++ b/runtime/runtime-inl.h
@@ -49,7 +49,9 @@
} else if (method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveRefsOnly)) {
return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveRefsOnly);
} else {
- DCHECK_EQ(method, GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverything));
+ DCHECK(method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverything) ||
+ method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverythingForClinit) ||
+ method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverythingForSuspendCheck));
return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveEverything);
}
}
diff --git a/runtime/runtime.h b/runtime/runtime.h
index 0c1344e..4e84468 100644
--- a/runtime/runtime.h
+++ b/runtime/runtime.h
@@ -714,7 +714,7 @@
static constexpr int kProfileForground = 0;
static constexpr int kProfileBackground = 1;
- static constexpr uint32_t kCalleeSaveSize = 4u;
+ static constexpr uint32_t kCalleeSaveSize = 6u;
// 64 bit so that we can share the same asm offsets for both 32 and 64 bits.
uint64_t callee_save_methods_[kCalleeSaveSize];
diff --git a/runtime/stack.cc b/runtime/stack.cc
index 19df0d2..2e06536 100644
--- a/runtime/stack.cc
+++ b/runtime/stack.cc
@@ -634,7 +634,7 @@
void StackVisitor::SanityCheckFrame() const {
if (kIsDebugBuild) {
ArtMethod* method = GetMethod();
- auto* declaring_class = method->GetDeclaringClass();
+ mirror::Class* declaring_class = method->GetDeclaringClass();
// Runtime methods have null declaring class.
if (!method->IsRuntimeMethod()) {
CHECK(declaring_class != nullptr);
@@ -647,11 +647,14 @@
LinearAlloc* const linear_alloc = runtime->GetLinearAlloc();
if (!linear_alloc->Contains(method)) {
// Check class linker linear allocs.
- mirror::Class* klass = method->GetDeclaringClass();
+ // We get the canonical method as copied methods may have their declaring
+ // class from another class loader.
+ ArtMethod* canonical = method->GetCanonicalMethod();
+ mirror::Class* klass = canonical->GetDeclaringClass();
LinearAlloc* const class_linear_alloc = (klass != nullptr)
? runtime->GetClassLinker()->GetAllocatorForClassLoader(klass->GetClassLoader())
: linear_alloc;
- if (!class_linear_alloc->Contains(method)) {
+ if (!class_linear_alloc->Contains(canonical)) {
// Check image space.
bool in_image = false;
for (auto& space : runtime->GetHeap()->GetContinuousSpaces()) {
@@ -660,14 +663,14 @@
const auto& header = image_space->GetImageHeader();
const ImageSection& methods = header.GetMethodsSection();
const ImageSection& runtime_methods = header.GetRuntimeMethodsSection();
- const size_t offset = reinterpret_cast<const uint8_t*>(method) - image_space->Begin();
+ const size_t offset = reinterpret_cast<const uint8_t*>(canonical) - image_space->Begin();
if (methods.Contains(offset) || runtime_methods.Contains(offset)) {
in_image = true;
break;
}
}
}
- CHECK(in_image) << method->PrettyMethod() << " not in linear alloc or image";
+ CHECK(in_image) << canonical->PrettyMethod() << " not in linear alloc or image";
}
}
if (cur_quick_frame_ != nullptr) {
diff --git a/runtime/thread.h b/runtime/thread.h
index 776096a..7540fd2 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -1705,8 +1705,13 @@
class SCOPED_CAPABILITY ScopedAssertNoThreadSuspension {
public:
- ALWAYS_INLINE explicit ScopedAssertNoThreadSuspension(const char* cause)
- ACQUIRE(Roles::uninterruptible_) {
+ ALWAYS_INLINE ScopedAssertNoThreadSuspension(const char* cause,
+ bool enabled = true)
+ ACQUIRE(Roles::uninterruptible_)
+ : enabled_(enabled) {
+ if (!enabled_) {
+ return;
+ }
if (kIsDebugBuild) {
self_ = Thread::Current();
old_cause_ = self_->StartAssertNoThreadSuspension(cause);
@@ -1715,6 +1720,9 @@
}
}
ALWAYS_INLINE ~ScopedAssertNoThreadSuspension() RELEASE(Roles::uninterruptible_) {
+ if (!enabled_) {
+ return;
+ }
if (kIsDebugBuild) {
self_->EndAssertNoThreadSuspension(old_cause_);
} else {
@@ -1724,6 +1732,7 @@
private:
Thread* self_;
+ const bool enabled_;
const char* old_cause_;
};
diff --git a/runtime/vdex_file.cc b/runtime/vdex_file.cc
index e8f947c..b955220 100644
--- a/runtime/vdex_file.cc
+++ b/runtime/vdex_file.cc
@@ -20,6 +20,7 @@
#include <memory>
+#include "base/bit_utils.h"
#include "base/logging.h"
#include "base/stl_util.h"
#include "base/unix_file/fd_file.h"
@@ -134,6 +135,9 @@
} else {
// Fetch the next dex file. Return null if there is none.
const uint8_t* data = cursor + reinterpret_cast<const DexFile::Header*>(cursor)->file_size_;
+ // Dex files are required to be 4 byte aligned. the OatWriter makes sure they are, see
+ // OatWriter::SeekToDexFiles.
+ data = AlignUp(data, 4);
return (data == DexEnd()) ? nullptr : data;
}
}
diff --git a/test/1338-gc-no-los/expected.txt b/test/1338-gc-no-los/expected.txt
new file mode 100644
index 0000000..36bec43
--- /dev/null
+++ b/test/1338-gc-no-los/expected.txt
@@ -0,0 +1 @@
+131072 200 true
diff --git a/test/1338-gc-no-los/info.txt b/test/1338-gc-no-los/info.txt
new file mode 100644
index 0000000..2e27357
--- /dev/null
+++ b/test/1338-gc-no-los/info.txt
@@ -0,0 +1 @@
+Test that the GC works with no large object space. Regression test for b/64393515.
\ No newline at end of file
diff --git a/test/1338-gc-no-los/run b/test/1338-gc-no-los/run
new file mode 100755
index 0000000..f3c43fb
--- /dev/null
+++ b/test/1338-gc-no-los/run
@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# Copyright 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+./default-run "$@" --runtime-option -XX:LargeObjectSpace=disabled
diff --git a/test/1338-gc-no-los/src-art/Main.java b/test/1338-gc-no-los/src-art/Main.java
new file mode 100644
index 0000000..662fa7e
--- /dev/null
+++ b/test/1338-gc-no-los/src-art/Main.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import dalvik.system.VMRuntime;
+import java.lang.ref.WeakReference;
+import java.lang.reflect.Method;
+
+public class Main {
+ public static void main(String[] args) {
+ try {
+ // Allocate a large object.
+ byte[] arr = new byte[128 * 1024];
+ // Allocate a non movable object.
+ byte[] arr2 = (byte[])VMRuntime.getRuntime().newNonMovableArray(Byte.TYPE, 200);
+ // Put the array in a weak reference so that IsMarked is called by the GC.
+ WeakReference weakRef = new WeakReference(arr2);
+ // Do a GC.
+ Runtime.getRuntime().gc();
+ arr[0] = 1;
+ arr2[0] = 1;
+ System.out.println(arr.length + " " + arr2.length + " " + (weakRef.get() != null));
+ } catch (Exception e) {
+ System.out.println(e);
+ }
+ }
+}
diff --git a/test/449-checker-bce/src/Main.java b/test/449-checker-bce/src/Main.java
index 5103540..f466eea 100644
--- a/test/449-checker-bce/src/Main.java
+++ b/test/449-checker-bce/src/Main.java
@@ -876,6 +876,91 @@
return true;
}
+ /// CHECK-START: void Main.modArrayIndex1(int[]) BCE (before)
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+
+ /// CHECK-START: void Main.modArrayIndex1(int[]) BCE (after)
+ /// CHECK-NOT: Deoptimize
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-NOT: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ public static void modArrayIndex1(int[] array) {
+ for(int i = 0; i < 100; i++) {
+ // Cannot statically eliminate, for example, when array.length == 5.
+ // Currently dynamic BCE isn't applied for this case.
+ array[i % 10] = i;
+ // Can be eliminated by BCE.
+ array[i % array.length] = i;
+ }
+ }
+
+ /// CHECK-START: void Main.modArrayIndex2(int[], int) BCE (before)
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+
+ /// CHECK-START: void Main.modArrayIndex2(int[], int) BCE (after)
+ /// CHECK-NOT: Deoptimize
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ public static void modArrayIndex2(int array[], int index) {
+ for(int i = 0; i < 100; i++) {
+ // Both bounds checks cannot be statically eliminated, because index can be < 0.
+ // Currently dynamic BCE isn't applied for this case.
+ array[(index+i) % 10] = i;
+ array[(index+i) % array.length] = i;
+ }
+ }
+
+ static final int[] staticArray = new int[10];
+
+ /// CHECK-START: void Main.modArrayIndex3() BCE (before)
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+
+ /// CHECK-START: void Main.modArrayIndex3() BCE (after)
+ /// CHECK-NOT: Deoptimize
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-NOT: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ public static void modArrayIndex3() {
+ for(int i = 0; i < 100; i++) {
+ // Currently dynamic BCE isn't applied for this case.
+ staticArray[i % 10] = i;
+ // Can be eliminated by BCE.
+ staticArray[i % staticArray.length] = i;
+ }
+ }
+
+ /// CHECK-START: void Main.modArrayIndex4() BCE (before)
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-DAG: BoundsCheck
+ /// CHECK-DAG: ArraySet
+
+ /// CHECK-START: void Main.modArrayIndex4() BCE (after)
+ /// CHECK-NOT: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ /// CHECK-NOT: BoundsCheck
+ /// CHECK-DAG: ArraySet
+ public static void modArrayIndex4() {
+ int[] array = new int[20];
+ for(int i = 0; i < 100; i++) {
+ // The local array length is statically know. Both can be eliminated by BCE.
+ array[i % 10] = i;
+ array[i % array.length] = i;
+ }
+ }
/// CHECK-START: void Main.bubbleSort(int[]) GVN (before)
/// CHECK: BoundsCheck
diff --git a/test/623-checker-loop-regressions/src/Main.java b/test/623-checker-loop-regressions/src/Main.java
index fc7bcb2..0e85612 100644
--- a/test/623-checker-loop-regressions/src/Main.java
+++ b/test/623-checker-loop-regressions/src/Main.java
@@ -441,6 +441,29 @@
}
}
+ /// CHECK-START: int Main.feedsIntoDeopt(int[]) loop_optimization (before)
+ /// CHECK-DAG: Phi loop:<<Loop1:B\d+>> outer_loop:none
+ /// CHECK-DAG: Phi loop:<<Loop1>> outer_loop:none
+ /// CHECK-DAG: Phi loop:<<Loop2:B\d+>> outer_loop:none
+ //
+ /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>"
+ //
+ /// CHECK-START: int Main.feedsIntoDeopt(int[]) loop_optimization (after)
+ /// CHECK-DAG: Phi loop:{{B\d+}} outer_loop:none
+ /// CHECK-NOT: Phi
+ static int feedsIntoDeopt(int[] a) {
+ // Reduction should be removed.
+ int r = 0;
+ for (int i = 0; i < 100; i++) {
+ r += 10;
+ }
+ // Even though uses feed into deopts of BCE.
+ for (int i = 1; i < 100; i++) {
+ a[i] = a[i - 1];
+ }
+ return r;
+ }
+
public static void main(String[] args) {
expectEquals(10, earlyExitFirst(-1));
for (int i = 0; i <= 10; i++) {
@@ -556,6 +579,13 @@
inductionMax(yy);
+ int[] f = new int[100];
+ f[0] = 11;
+ expectEquals(1000, feedsIntoDeopt(f));
+ for (int i = 0; i < 100; i++) {
+ expectEquals(11, f[i]);
+ }
+
System.out.println("passed");
}
diff --git a/test/661-checker-simd-reduc/expected.txt b/test/661-checker-simd-reduc/expected.txt
new file mode 100644
index 0000000..b0aad4d
--- /dev/null
+++ b/test/661-checker-simd-reduc/expected.txt
@@ -0,0 +1 @@
+passed
diff --git a/test/661-checker-simd-reduc/info.txt b/test/661-checker-simd-reduc/info.txt
new file mode 100644
index 0000000..4d071fb
--- /dev/null
+++ b/test/661-checker-simd-reduc/info.txt
@@ -0,0 +1 @@
+Functional tests on vectorization of the most basic reductions.
diff --git a/test/661-checker-simd-reduc/src/Main.java b/test/661-checker-simd-reduc/src/Main.java
new file mode 100644
index 0000000..741b5fa
--- /dev/null
+++ b/test/661-checker-simd-reduc/src/Main.java
@@ -0,0 +1,292 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Tests for simple integral reductions: same type for accumulator and data.
+ */
+public class Main {
+
+ static final int N = 500;
+
+ //
+ // Basic reductions in loops.
+ //
+
+ // TODO: vectorize these (second step of b/64091002 plan)
+
+ private static byte reductionByte(byte[] x) {
+ byte sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum += x[i];
+ }
+ return sum;
+ }
+
+ private static short reductionShort(short[] x) {
+ short sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum += x[i];
+ }
+ return sum;
+ }
+
+ private static char reductionChar(char[] x) {
+ char sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum += x[i];
+ }
+ return sum;
+ }
+
+ private static int reductionInt(int[] x) {
+ int sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum += x[i];
+ }
+ return sum;
+ }
+
+ private static long reductionLong(long[] x) {
+ long sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum += x[i];
+ }
+ return sum;
+ }
+
+ private static byte reductionMinusByte(byte[] x) {
+ byte sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum -= x[i];
+ }
+ return sum;
+ }
+
+ private static short reductionMinusShort(short[] x) {
+ short sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum -= x[i];
+ }
+ return sum;
+ }
+
+ private static char reductionMinusChar(char[] x) {
+ char sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum -= x[i];
+ }
+ return sum;
+ }
+
+ private static int reductionMinusInt(int[] x) {
+ int sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum -= x[i];
+ }
+ return sum;
+ }
+
+ private static long reductionMinusLong(long[] x) {
+ long sum = 0;
+ for (int i = 0; i < x.length; i++) {
+ sum -= x[i];
+ }
+ return sum;
+ }
+
+ private static byte reductionMinByte(byte[] x) {
+ byte min = Byte.MAX_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ min = (byte) Math.min(min, x[i]);
+ }
+ return min;
+ }
+
+ private static short reductionMinShort(short[] x) {
+ short min = Short.MAX_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ min = (short) Math.min(min, x[i]);
+ }
+ return min;
+ }
+
+ private static char reductionMinChar(char[] x) {
+ char min = Character.MAX_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ min = (char) Math.min(min, x[i]);
+ }
+ return min;
+ }
+
+ private static int reductionMinInt(int[] x) {
+ int min = Integer.MAX_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ min = Math.min(min, x[i]);
+ }
+ return min;
+ }
+
+ private static long reductionMinLong(long[] x) {
+ long min = Long.MAX_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ min = Math.min(min, x[i]);
+ }
+ return min;
+ }
+
+ private static byte reductionMaxByte(byte[] x) {
+ byte max = Byte.MIN_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ max = (byte) Math.max(max, x[i]);
+ }
+ return max;
+ }
+
+ private static short reductionMaxShort(short[] x) {
+ short max = Short.MIN_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ max = (short) Math.max(max, x[i]);
+ }
+ return max;
+ }
+
+ private static char reductionMaxChar(char[] x) {
+ char max = Character.MIN_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ max = (char) Math.max(max, x[i]);
+ }
+ return max;
+ }
+
+ private static int reductionMaxInt(int[] x) {
+ int max = Integer.MIN_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ max = Math.max(max, x[i]);
+ }
+ return max;
+ }
+
+ private static long reductionMaxLong(long[] x) {
+ long max = Long.MIN_VALUE;
+ for (int i = 0; i < x.length; i++) {
+ max = Math.max(max, x[i]);
+ }
+ return max;
+ }
+
+ //
+ // A few special cases.
+ //
+
+ // TODO: consider unrolling
+
+ private static int reductionInt10(int[] x) {
+ int sum = 0;
+ // Amenable to complete unrolling.
+ for (int i = 10; i <= 10; i++) {
+ sum += x[i];
+ }
+ return sum;
+ }
+
+ private static int reductionMinusInt10(int[] x) {
+ int sum = 0;
+ // Amenable to complete unrolling.
+ for (int i = 10; i <= 10; i++) {
+ sum -= x[i];
+ }
+ return sum;
+ }
+
+ private static int reductionMinInt10(int[] x) {
+ int min = Integer.MAX_VALUE;
+ // Amenable to complete unrolling.
+ for (int i = 10; i <= 10; i++) {
+ min = Math.min(min, x[i]);
+ }
+ return min;
+ }
+
+ private static int reductionMaxInt10(int[] x) {
+ int max = Integer.MIN_VALUE;
+ // Amenable to complete unrolling.
+ for (int i = 10; i <= 10; i++) {
+ max = Math.max(max, x[i]);
+ }
+ return max;
+ }
+
+ //
+ // Main driver.
+ //
+
+ public static void main(String[] args) {
+ byte[] xb = new byte[N];
+ short[] xs = new short[N];
+ char[] xc = new char[N];
+ int[] xi = new int[N];
+ long[] xl = new long[N];
+ for (int i = 0, k = -17; i < N; i++, k += 3) {
+ xb[i] = (byte) k;
+ xs[i] = (short) k;
+ xc[i] = (char) k;
+ xi[i] = k;
+ xl[i] = k;
+ }
+
+ // Test various reductions in loops.
+ expectEquals(-74, reductionByte(xb));
+ expectEquals(-27466, reductionShort(xs));
+ expectEquals(38070, reductionChar(xc));
+ expectEquals(365750, reductionInt(xi));
+ expectEquals(365750L, reductionLong(xl));
+ expectEquals(74, reductionMinusByte(xb));
+ expectEquals(27466, reductionMinusShort(xs));
+ expectEquals(27466, reductionMinusChar(xc));
+ expectEquals(-365750, reductionMinusInt(xi));
+ expectEquals(-365750L, reductionMinusLong(xl));
+ expectEquals(-128, reductionMinByte(xb));
+ expectEquals(-17, reductionMinShort(xs));
+ expectEquals(1, reductionMinChar(xc));
+ expectEquals(-17, reductionMinInt(xi));
+ expectEquals(-17L, reductionMinLong(xl));
+ expectEquals(127, reductionMaxByte(xb));
+ expectEquals(1480, reductionMaxShort(xs));
+ expectEquals(65534, reductionMaxChar(xc));
+ expectEquals(1480, reductionMaxInt(xi));
+ expectEquals(1480L, reductionMaxLong(xl));
+
+ // Test special cases.
+ expectEquals(13, reductionInt10(xi));
+ expectEquals(-13, reductionMinusInt10(xi));
+ expectEquals(13, reductionMinInt10(xi));
+ expectEquals(13, reductionMaxInt10(xi));
+
+ System.out.println("passed");
+ }
+
+ private static void expectEquals(int expected, int result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ private static void expectEquals(long expected, long result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+}
diff --git a/test/661-classloader-allocator/expected.txt b/test/661-classloader-allocator/expected.txt
new file mode 100644
index 0000000..6a5618e
--- /dev/null
+++ b/test/661-classloader-allocator/expected.txt
@@ -0,0 +1 @@
+JNI_OnLoad called
diff --git a/test/661-classloader-allocator/info.txt b/test/661-classloader-allocator/info.txt
new file mode 100644
index 0000000..872b5e0
--- /dev/null
+++ b/test/661-classloader-allocator/info.txt
@@ -0,0 +1,3 @@
+Regression test for copied methods which used to not
+be (re-)allocated with the right class loader allocator,
+which crashed the JIT profiler.
diff --git a/test/661-classloader-allocator/src-ex/OtherClass.java b/test/661-classloader-allocator/src-ex/OtherClass.java
new file mode 100644
index 0000000..e59cb95
--- /dev/null
+++ b/test/661-classloader-allocator/src-ex/OtherClass.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package p1;
+
+interface Itf {
+ public default int defaultMethod() {
+ return 42;
+ }
+}
+
+public class OtherClass implements Itf {
+ public void foo() {
+ }
+}
diff --git a/test/661-classloader-allocator/src/Main.java b/test/661-classloader-allocator/src/Main.java
new file mode 100644
index 0000000..40f8f7a
--- /dev/null
+++ b/test/661-classloader-allocator/src/Main.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.reflect.Constructor;
+
+public class Main {
+ static final String DEX_FILE =
+ System.getenv("DEX_LOCATION") + "/661-classloader-allocator-ex.jar";
+ static final String LIBRARY_SEARCH_PATH = System.getProperty("java.library.path");
+
+ private static void doUnloading() {
+ // Stop the JIT to ensure its threads and work queue are not keeping classes
+ // artifically alive.
+ stopJit();
+ // Do multiple GCs to prevent rare flakiness if some other thread is keeping the
+ // classloader live.
+ for (int i = 0; i < 5; ++i) {
+ Runtime.getRuntime().gc();
+ }
+ startJit();
+ }
+
+ public static void main(String[] args) throws Exception {
+ System.loadLibrary(args[0]);
+ loadClass();
+ doUnloading();
+ // fetchProfiles iterate over the ProfilingInfo, we used to crash in the presence
+ // of unloaded copied methods.
+ fetchProfiles();
+ }
+
+ public static void loadClass() throws Exception {
+ Class<?> pathClassLoader = Class.forName("dalvik.system.PathClassLoader");
+ if (pathClassLoader == null) {
+ throw new AssertionError("Couldn't find path class loader class");
+ }
+ Constructor<?> constructor =
+ pathClassLoader.getDeclaredConstructor(String.class, String.class, ClassLoader.class);
+ ClassLoader loader = (ClassLoader) constructor.newInstance(
+ DEX_FILE, LIBRARY_SEARCH_PATH, ClassLoader.getSystemClassLoader());
+ Class<?> otherClass = loader.loadClass("p1.OtherClass");
+ ensureJitCompiled(otherClass, "foo");
+ }
+
+ public static native void ensureJitCompiled(Class<?> cls, String methodName);
+ public static native void fetchProfiles();
+ public static native void stopJit();
+ public static native void startJit();
+}
diff --git a/test/662-regression-alias/expected.txt b/test/662-regression-alias/expected.txt
new file mode 100644
index 0000000..7250211
--- /dev/null
+++ b/test/662-regression-alias/expected.txt
@@ -0,0 +1 @@
+passed 127 4
diff --git a/test/662-regression-alias/info.txt b/test/662-regression-alias/info.txt
new file mode 100644
index 0000000..584d3ee
--- /dev/null
+++ b/test/662-regression-alias/info.txt
@@ -0,0 +1 @@
+Regression test on missed array element aliasing.
diff --git a/test/662-regression-alias/src/Main.java b/test/662-regression-alias/src/Main.java
new file mode 100644
index 0000000..aad61e2
--- /dev/null
+++ b/test/662-regression-alias/src/Main.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Regression test on ARM-scheduling/array-aliasing bug (b/64018485).
+ */
+public class Main {
+
+ //
+ // Mimic original bug.
+ //
+
+ static void setFields(int[] fields) {
+ if (fields == null || fields.length < 6)
+ fields = new int[6]; // creates phi
+ fields[5] = 127;
+ }
+
+ static void processFieldValues(int field0, int field1, int field2,
+ int field3, int field4, int field5) {
+ if (field5 != 127) {
+ throw new Error("field = " + field5);
+ } else if (field0 != 0) {
+ processFieldValues(0, 0, 0, 0, 0, 0); // disable inlining
+ }
+ }
+
+ static int doit(int pass) {
+ int[] fields = new int[6];
+ for (; ; pass++) {
+ setFields(fields);
+ processFieldValues(fields[0], fields[1], fields[2],
+ fields[3], fields[4], fields[5]);
+ if (pass == 0)
+ break;
+ }
+ return fields[5];
+ }
+
+ //
+ // Similar situation.
+ //
+
+ private static int aliasing(boolean f) {
+ int[] array = new int[6];
+ int[] array2 = null;
+ int s = 0;
+ for (int i = 0; i < 1; i++) {
+ if (f) {
+ array2 = array;
+ }
+ array2[1] = 4;
+ s = array[1];
+ }
+ return s;
+ }
+
+ //
+ // Main driver.
+ //
+
+ static public void main(String[] args) {
+ int r = doit(0);
+ int s = aliasing(true);
+ System.out.println("passed " + r + " " + s);
+ }
+}
diff --git a/test/663-odd-dex-size/classes.dex b/test/663-odd-dex-size/classes.dex
new file mode 100644
index 0000000..633e3a2
--- /dev/null
+++ b/test/663-odd-dex-size/classes.dex
Binary files differ
diff --git a/test/663-odd-dex-size/expected.txt b/test/663-odd-dex-size/expected.txt
new file mode 100644
index 0000000..3da1ec2
--- /dev/null
+++ b/test/663-odd-dex-size/expected.txt
@@ -0,0 +1 @@
+HelloWorld
diff --git a/test/663-odd-dex-size/info.txt b/test/663-odd-dex-size/info.txt
new file mode 100644
index 0000000..11a50e0
--- /dev/null
+++ b/test/663-odd-dex-size/info.txt
@@ -0,0 +1,14 @@
+Test for a dex file with an odd size in a vdex file.
+
+The code in the file is:
+
+class Main {
+ public static void main(String[] args) {
+ System.out.println("HelloWorld");
+ }
+}
+
+The generated dex file was then manually edited to:
+1) Add 1 to the size value in the dex header.
+2) Add 1 byte to the file.
+3) Change the checksum in the dex header.
diff --git a/test/663-odd-dex-size2/663-odd-dex-size2.jar b/test/663-odd-dex-size2/663-odd-dex-size2.jar
new file mode 100644
index 0000000..a29224e
--- /dev/null
+++ b/test/663-odd-dex-size2/663-odd-dex-size2.jar
Binary files differ
diff --git a/test/663-odd-dex-size2/build b/test/663-odd-dex-size2/build
new file mode 100644
index 0000000..5636558
--- /dev/null
+++ b/test/663-odd-dex-size2/build
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Nothing to do
diff --git a/test/663-odd-dex-size2/expected.txt b/test/663-odd-dex-size2/expected.txt
new file mode 100644
index 0000000..3da1ec2
--- /dev/null
+++ b/test/663-odd-dex-size2/expected.txt
@@ -0,0 +1 @@
+HelloWorld
diff --git a/test/663-odd-dex-size2/info.txt b/test/663-odd-dex-size2/info.txt
new file mode 100644
index 0000000..900394d
--- /dev/null
+++ b/test/663-odd-dex-size2/info.txt
@@ -0,0 +1,15 @@
+Test for two files with an odd size in a vdex file.
+
+The code in boths file is:
+
+class Main {
+ public static void main(String[] args) {
+ System.out.println("HelloWorld");
+ }
+}
+
+The generated dex file was then manually edited to:
+1) Add 1 to the size value in the dex header.
+2) Add 1 byte to the file.
+3) Change the checksum in the dex header.
+
diff --git a/test/663-odd-dex-size3/663-odd-dex-size3.jar b/test/663-odd-dex-size3/663-odd-dex-size3.jar
new file mode 100644
index 0000000..d23ed57
--- /dev/null
+++ b/test/663-odd-dex-size3/663-odd-dex-size3.jar
Binary files differ
diff --git a/test/663-odd-dex-size3/build b/test/663-odd-dex-size3/build
new file mode 100644
index 0000000..5636558
--- /dev/null
+++ b/test/663-odd-dex-size3/build
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Nothing to do
diff --git a/test/663-odd-dex-size3/expected.txt b/test/663-odd-dex-size3/expected.txt
new file mode 100644
index 0000000..3da1ec2
--- /dev/null
+++ b/test/663-odd-dex-size3/expected.txt
@@ -0,0 +1 @@
+HelloWorld
diff --git a/test/663-odd-dex-size3/info.txt b/test/663-odd-dex-size3/info.txt
new file mode 100644
index 0000000..256c77d
--- /dev/null
+++ b/test/663-odd-dex-size3/info.txt
@@ -0,0 +1,19 @@
+Test for a dex file with an odd size followed by an aligned dex file.
+
+The code in classes.dex is:
+
+class Main {
+ public static void main(String[] args) {
+ System.out.println("HelloWorld");
+ }
+}
+
+The generated dex file was then manually edited to:
+1) Add 1 to the size value in the dex header.
+2) Add 1 byte to the file.
+3) Change the checksum in the dex header.
+
+The code in classes2.dex is:
+
+class Foo {
+}
diff --git a/test/663-odd-dex-size4/663-odd-dex-size4.jar b/test/663-odd-dex-size4/663-odd-dex-size4.jar
new file mode 100644
index 0000000..d229663
--- /dev/null
+++ b/test/663-odd-dex-size4/663-odd-dex-size4.jar
Binary files differ
diff --git a/test/663-odd-dex-size4/build b/test/663-odd-dex-size4/build
new file mode 100644
index 0000000..5636558
--- /dev/null
+++ b/test/663-odd-dex-size4/build
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Nothing to do
diff --git a/test/663-odd-dex-size4/expected.txt b/test/663-odd-dex-size4/expected.txt
new file mode 100644
index 0000000..3da1ec2
--- /dev/null
+++ b/test/663-odd-dex-size4/expected.txt
@@ -0,0 +1 @@
+HelloWorld
diff --git a/test/663-odd-dex-size4/info.txt b/test/663-odd-dex-size4/info.txt
new file mode 100644
index 0000000..2c34557
--- /dev/null
+++ b/test/663-odd-dex-size4/info.txt
@@ -0,0 +1,19 @@
+Test for an aligned dex file followed by a dex file with an odd size.
+
+The code in classes.dex is:
+
+class Foo {
+}
+
+The code in classes2.dex is:
+
+class Main {
+ public static void main(String[] args) {
+ System.out.println("HelloWorld");
+ }
+}
+
+The generated dex file was then manually edited to:
+1) Add 1 to the size value in the dex header.
+2) Add 1 byte to the file.
+3) Change the checksum in the dex header.
diff --git a/test/706-checker-scheduler/run b/test/706-checker-scheduler/run
new file mode 100644
index 0000000..5ffc303
--- /dev/null
+++ b/test/706-checker-scheduler/run
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use secondary switch to add secondary dex file to class path.
+exec ${RUN} "${@}" --secondary
diff --git a/test/706-checker-scheduler/src-dex2oat-unresolved/UnresolvedClass.java b/test/706-checker-scheduler/src-dex2oat-unresolved/UnresolvedClass.java
new file mode 100644
index 0000000..4faa12a
--- /dev/null
+++ b/test/706-checker-scheduler/src-dex2oat-unresolved/UnresolvedClass.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class UnresolvedClass {
+ public static int staticInt;
+ public int instanceInt;
+}
+
diff --git a/test/706-checker-scheduler/src/Main.java b/test/706-checker-scheduler/src/Main.java
index a68565b..08a23a7 100644
--- a/test/706-checker-scheduler/src/Main.java
+++ b/test/706-checker-scheduler/src/Main.java
@@ -31,6 +31,7 @@
public ExampleObj my_obj;
public static int number1;
public static int number2;
+ public static volatile int number3;
/// CHECK-START-ARM64: int Main.arrayAccess() scheduler (before)
/// CHECK: <<Const1:i\d+>> IntConstant 1
@@ -340,6 +341,87 @@
}
}
+ /// CHECK-START-ARM: void Main.accessFieldsVolatile() scheduler (before)
+ /// CHECK-START-ARM64: void Main.accessFieldsVolatile() scheduler (before)
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: StaticFieldGet
+ /// CHECK: Add
+ /// CHECK: StaticFieldSet
+ /// CHECK: StaticFieldGet
+ /// CHECK: Add
+ /// CHECK: StaticFieldSet
+
+ /// CHECK-START-ARM: void Main.accessFieldsVolatile() scheduler (after)
+ /// CHECK-START-ARM64: void Main.accessFieldsVolatile() scheduler (after)
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: StaticFieldGet
+ /// CHECK: Add
+ /// CHECK: StaticFieldSet
+ /// CHECK: StaticFieldGet
+ /// CHECK: Add
+ /// CHECK: StaticFieldSet
+
+ public void accessFieldsVolatile() {
+ my_obj = new ExampleObj(1, 2);
+ for (int i = 0; i < 10; i++) {
+ my_obj.n1++;
+ my_obj.n2++;
+ number1++;
+ number3++;
+ }
+ }
+
+ /// CHECK-START-ARM: void Main.accessFieldsUnresolved() scheduler (before)
+ /// CHECK-START-ARM64: void Main.accessFieldsUnresolved() scheduler (before)
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: UnresolvedInstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: UnresolvedInstanceFieldSet
+ /// CHECK: UnresolvedStaticFieldGet
+ /// CHECK: Add
+ /// CHECK: UnresolvedStaticFieldSet
+
+ /// CHECK-START-ARM: void Main.accessFieldsUnresolved() scheduler (after)
+ /// CHECK-START-ARM64: void Main.accessFieldsUnresolved() scheduler (after)
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: InstanceFieldSet
+ /// CHECK: UnresolvedInstanceFieldGet
+ /// CHECK: Add
+ /// CHECK: UnresolvedInstanceFieldSet
+ /// CHECK: UnresolvedStaticFieldGet
+ /// CHECK: Add
+ /// CHECK: UnresolvedStaticFieldSet
+
+ public void accessFieldsUnresolved() {
+ my_obj = new ExampleObj(1, 2);
+ UnresolvedClass unresolved_obj = new UnresolvedClass();
+ for (int i = 0; i < 10; i++) {
+ my_obj.n1++;
+ my_obj.n2++;
+ unresolved_obj.instanceInt++;
+ UnresolvedClass.staticInt++;
+ }
+ }
+
/// CHECK-START-ARM64: int Main.intDiv(int) scheduler (before)
/// CHECK: Sub
/// CHECK: DivZeroCheck
diff --git a/test/common/runtime_state.cc b/test/common/runtime_state.cc
index 7c0ed69..60c7315 100644
--- a/test/common/runtime_state.cc
+++ b/test/common/runtime_state.cc
@@ -254,4 +254,17 @@
return Runtime::Current()->GetNumberOfDeoptimizations();
}
+extern "C" JNIEXPORT void JNICALL Java_Main_fetchProfiles(JNIEnv*, jclass) {
+ jit::Jit* jit = GetJitIfEnabled();
+ if (jit == nullptr) {
+ return;
+ }
+ jit::JitCodeCache* code_cache = jit->GetCodeCache();
+ std::vector<ProfileMethodInfo> unused_vector;
+ std::set<std::string> unused_locations;
+ unused_locations.insert("fake_location");
+ ScopedObjectAccess soa(Thread::Current());
+ code_cache->GetProfiledMethods(unused_locations, unused_vector);
+}
+
} // namespace art
diff --git a/test/knownfailures.json b/test/knownfailures.json
index bd9591a..a8191bb 100644
--- a/test/knownfailures.json
+++ b/test/knownfailures.json
@@ -713,8 +713,16 @@
"--no-dex2oat do not create"]
},
{
- "tests": "961-default-iface-resolution-gen",
+ "tests": ["961-default-iface-resolution-gen",
+ "964-default-iface-init-gen",
+ "968-default-partial-compile-gen"],
"env_vars": {"SANITIZE_HOST": "address"},
"description": ["Test hits dex2oat watchdog timeout (60sec) on art-asan"]
+ },
+ {
+ "tests": "662-regression-alias",
+ "variant": "target",
+ "description": ["disable until ARM scheduling/aliasing bug is fixed."],
+ "bug": "b/64018485"
}
]
diff --git a/test/ti-agent/trace_helper.cc b/test/ti-agent/trace_helper.cc
index 7a9d1e0..1f8ceff 100644
--- a/test/ti-agent/trace_helper.cc
+++ b/test/ti-agent/trace_helper.cc
@@ -388,10 +388,12 @@
data->single_step = single_step != nullptr ? env->FromReflectedMethod(single_step) : nullptr;
data->in_callback = false;
- void* old_data = nullptr;
- if (JvmtiErrorToException(env, jvmti_env, jvmti_env->GetEnvironmentLocalStorage(&old_data))) {
+ TraceData* old_data = nullptr;
+ if (JvmtiErrorToException(env, jvmti_env,
+ jvmti_env->GetEnvironmentLocalStorage(
+ reinterpret_cast<void**>(&old_data)))) {
return;
- } else if (old_data != nullptr) {
+ } else if (old_data != nullptr && old_data->test_klass != nullptr) {
ScopedLocalRef<jclass> rt_exception(env, env->FindClass("java/lang/RuntimeException"));
env->ThrowNew(rt_exception.get(), "Environment already has local storage set!");
return;
@@ -455,6 +457,21 @@
extern "C" JNIEXPORT void JNICALL Java_art_Trace_disableTracing(
JNIEnv* env, jclass klass ATTRIBUTE_UNUSED, jthread thr) {
+ TraceData* data = nullptr;
+ if (JvmtiErrorToException(
+ env, jvmti_env, jvmti_env->GetEnvironmentLocalStorage(reinterpret_cast<void**>(&data)))) {
+ return;
+ }
+ // If data is null then we haven't ever enabled tracing so we don't need to do anything.
+ if (data == nullptr || data->test_klass == nullptr) {
+ return;
+ }
+ env->DeleteGlobalRef(data->test_klass);
+ if (env->ExceptionCheck()) {
+ return;
+ }
+ // Clear test_klass so we know this isn't being used
+ data->test_klass = nullptr;
if (JvmtiErrorToException(env, jvmti_env,
jvmti_env->SetEventNotificationMode(JVMTI_DISABLE,
JVMTI_EVENT_FIELD_ACCESS,
diff --git a/tools/art b/tools/art
index dea7ebd..15993dd 100644
--- a/tools/art
+++ b/tools/art
@@ -23,6 +23,7 @@
LIBART=libart.so
JIT_PROFILE="no"
VERBOSE="no"
+CLEAN_OAT_FILES="yes"
EXTRA_OPTIONS=()
# Follow all sym links to get the program name.
@@ -87,6 +88,7 @@
report upon completion.
--profile Run with profiling, then run using profile data.
--verbose Run script verbosely.
+ --no-clean Don't cleanup oat directories.
The ART_OPTIONS are passed directly to the Android Runtime.
@@ -176,12 +178,14 @@
# Input: Command line arguments to the art script.
# e.g. -cp foo/classes.dex:bar/classes.dex would delete (foo/oat bar/oat) directories.
cleanup_oat_directory_for_classpath() {
- # First try: Use $CLASSPATH environment variable.
- parse_classpath "$CLASSPATH"
- # Second try: Look for latest -cp or -classpath arg which will take precedence.
- find_cp_in_args "$@"
+ if [ "$CLEAN_OAT_FILES" = "yes" ]; then
+ # First try: Use $CLASSPATH environment variable.
+ parse_classpath "$CLASSPATH"
+ # Second try: Look for latest -cp or -classpath arg which will take precedence.
+ find_cp_in_args "$@"
- cleanup_oat_directory "${PARSE_CLASSPATH_RESULT[@]}"
+ cleanup_oat_directory "${PARSE_CLASSPATH_RESULT[@]}"
+ fi
}
# Attempt to find $ANDROID_ROOT/framework/<isa>/core.art' without knowing what <isa> is.
@@ -291,6 +295,9 @@
--verbose)
VERBOSE="yes"
;;
+ --no-clean)
+ CLEAN_OAT_FILES="no"
+ ;;
--*)
echo "unknown option: $1" 1>&2
usage
diff --git a/tools/cpp-define-generator/offset_runtime.def b/tools/cpp-define-generator/offset_runtime.def
index 41e7e40..1d5ce7d 100644
--- a/tools/cpp-define-generator/offset_runtime.def
+++ b/tools/cpp-define-generator/offset_runtime.def
@@ -40,6 +40,10 @@
DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_REFS_AND_ARGS, art::CalleeSaveType::kSaveRefsAndArgs)
// Offset of field Runtime::callee_save_methods_[kSaveEverything]
DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING, art::CalleeSaveType::kSaveEverything)
+// Offset of field Runtime::callee_save_methods_[kSaveEverythingForClinit]
+DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING_FOR_CLINIT, art::CalleeSaveType::kSaveEverythingForClinit)
+// Offset of field Runtime::callee_save_methods_[kSaveEverythingForSuspendCheck]
+DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING_FOR_SUSPEND_CHECK, art::CalleeSaveType::kSaveEverythingForSuspendCheck)
#undef DEFINE_RUNTIME_CALLEE_SAVE_OFFSET
#include "common_undef.def" // undef DEFINE_OFFSET_EXPR
diff --git a/tools/dexfuzz/README b/tools/dexfuzz/README
index 1f74262..fff5473 100644
--- a/tools/dexfuzz/README
+++ b/tools/dexfuzz/README
@@ -139,6 +139,7 @@
InstructionSwapper 80
InvokeChanger 30
NewArrayLengthChanger 50
+NewInstanceChanger 10
NewMethodCaller 10
NonsenseStringPrinter 10
OppositeBranchChanger 40
diff --git a/tools/dexfuzz/src/dexfuzz/DexFuzz.java b/tools/dexfuzz/src/dexfuzz/DexFuzz.java
index 2b3b8e7..1e37def 100644
--- a/tools/dexfuzz/src/dexfuzz/DexFuzz.java
+++ b/tools/dexfuzz/src/dexfuzz/DexFuzz.java
@@ -33,9 +33,9 @@
* Entrypoint class for dexfuzz.
*/
public class DexFuzz {
- // Last version update 1.7: changed the likelihood of RegisterClobber.
+ // Last version update 1.8: Added a new mutation called NewInstanceChanger.
private static int majorVersion = 1;
- private static int minorVersion = 7;
+ private static int minorVersion = 8;
private static int seedChangeVersion = 0;
/**
diff --git a/tools/dexfuzz/src/dexfuzz/program/Program.java b/tools/dexfuzz/src/dexfuzz/program/Program.java
index bb2f4c0..c6fa6c4 100644
--- a/tools/dexfuzz/src/dexfuzz/program/Program.java
+++ b/tools/dexfuzz/src/dexfuzz/program/Program.java
@@ -32,6 +32,7 @@
import dexfuzz.program.mutators.InstructionSwapper;
import dexfuzz.program.mutators.InvokeChanger;
import dexfuzz.program.mutators.NewArrayLengthChanger;
+import dexfuzz.program.mutators.NewInstanceChanger;
import dexfuzz.program.mutators.NewMethodCaller;
import dexfuzz.program.mutators.NonsenseStringPrinter;
import dexfuzz.program.mutators.OppositeBranchChanger;
@@ -54,6 +55,7 @@
import dexfuzz.rawdex.ProtoIdItem;
import dexfuzz.rawdex.RawDexFile;
import dexfuzz.rawdex.TypeIdItem;
+import dexfuzz.rawdex.TypeList;
import dexfuzz.rawdex.formats.ContainsPoolIndex.PoolIndexKind;
import java.io.BufferedReader;
@@ -204,6 +206,7 @@
registerMutator(new InstructionSwapper(rng, mutationStats, mutations));
registerMutator(new InvokeChanger(rng, mutationStats, mutations));
registerMutator(new NewArrayLengthChanger(rng, mutationStats, mutations));
+ registerMutator(new NewInstanceChanger(rng, mutationStats, mutations));
registerMutator(new NewMethodCaller(rng, mutationStats, mutations));
registerMutator(new NonsenseStringPrinter(rng, mutationStats, mutations));
registerMutator(new OppositeBranchChanger(rng, mutationStats, mutations));
@@ -609,4 +612,45 @@
fieldIdx));
return null;
}
-}
+
+ /**
+ * Used to convert the type index into string format.
+ * @param typeIdx
+ * @return string format of type index.
+ */
+ public String getTypeString(int typeIdx) {
+ TypeIdItem typeIdItem = rawDexFile.typeIds.get(typeIdx);
+ return rawDexFile.stringDatas.get(typeIdItem.descriptorIdx).getString();
+ }
+
+ /**
+ * Used to convert the method index into string format.
+ * @param methodIdx
+ * @return string format of method index.
+ */
+ public String getMethodString(int methodIdx) {
+ MethodIdItem methodIdItem = rawDexFile.methodIds.get(methodIdx);
+ return rawDexFile.stringDatas.get(methodIdItem.nameIdx).getString();
+ }
+
+ /**
+ * Used to convert methodID to string format of method proto.
+ * @param methodIdx
+ * @return string format of shorty.
+ */
+ public String getMethodProto(int methodIdx) {
+ MethodIdItem methodIdItem = rawDexFile.methodIds.get(methodIdx);
+ ProtoIdItem protoIdItem = rawDexFile.protoIds.get(methodIdItem.protoIdx);
+
+ if (!protoIdItem.parametersOff.pointsToSomething()) {
+ return "()" + getTypeString(protoIdItem.returnTypeIdx);
+ }
+
+ TypeList typeList = (TypeList) protoIdItem.parametersOff.getPointedToItem();
+ String typeItem = "(";
+ for (int i= 0; i < typeList.size; i++) {
+ typeItem = typeItem + typeList.list[i];
+ }
+ return typeItem + ")" + getTypeString(protoIdItem.returnTypeIdx);
+ }
+}
\ No newline at end of file
diff --git a/tools/dexfuzz/src/dexfuzz/program/mutators/NewInstanceChanger.java b/tools/dexfuzz/src/dexfuzz/program/mutators/NewInstanceChanger.java
new file mode 100644
index 0000000..cbf79e3
--- /dev/null
+++ b/tools/dexfuzz/src/dexfuzz/program/mutators/NewInstanceChanger.java
@@ -0,0 +1,218 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dexfuzz.program.mutators;
+
+import dexfuzz.Log;
+import dexfuzz.MutationStats;
+import dexfuzz.program.MInsn;
+import dexfuzz.program.MutatableCode;
+import dexfuzz.program.Mutation;
+import dexfuzz.rawdex.Opcode;
+import dexfuzz.rawdex.formats.ContainsPoolIndex;
+import dexfuzz.rawdex.formats.ContainsPoolIndex.PoolIndexKind;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Mutator NewInstanceChanger changes the new instance type in a method to
+ * any random type from the pool.
+ */
+public class NewInstanceChanger extends CodeMutator {
+
+ /**
+ * Every CodeMutator has an AssociatedMutation, representing the
+ * mutation that this CodeMutator can perform, to allow separate
+ * generateMutation() and applyMutation() phases, allowing serialization.
+ */
+ public static class AssociatedMutation extends Mutation {
+ public int newInstanceToChangeIdx;
+ public int newInstanceTypeIdx;
+
+ @Override
+ public String getString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(newInstanceToChangeIdx).append(" ");
+ builder.append(newInstanceTypeIdx);
+ return builder.toString();
+ }
+
+ @Override
+ public void parseString(String[] elements) {
+ newInstanceToChangeIdx = Integer.parseInt(elements[2]);
+ newInstanceTypeIdx = Integer.parseInt(elements[3]);
+ }
+ }
+
+ // The following two methods are here for the benefit of MutationSerializer,
+ // so it can create a CodeMutator and get the correct associated Mutation, as it
+ // reads in mutations from a dump of mutations.
+ @Override
+ public Mutation getNewMutation() {
+ return new AssociatedMutation();
+ }
+
+ public NewInstanceChanger() {}
+
+ public NewInstanceChanger(Random rng, MutationStats stats, List<Mutation> mutations) {
+ super(rng, stats, mutations);
+ likelihood = 10;
+ }
+
+ // A cache that should only exist between generateMutation() and applyMutation(),
+ // or be created at the start of applyMutation(), if we're reading in mutations from
+ // a file.
+ private List<MInsn> newInstanceCachedInsns = null;
+
+ private void generateCachedNewInstanceInsns(MutatableCode mutatableCode) {
+ if (newInstanceCachedInsns != null) {
+ return;
+ }
+
+ newInstanceCachedInsns = new ArrayList<MInsn>();
+
+ for (MInsn mInsn : mutatableCode.getInstructions()) {
+ if (mInsn.insn.info.opcode == Opcode.NEW_INSTANCE) {
+ newInstanceCachedInsns.add(mInsn);
+ }
+ }
+ }
+
+ @Override
+ protected boolean canMutate(MutatableCode mutatableCode) {
+ // Cannot change the pool index with only one type.
+ if (mutatableCode.program.getTotalPoolIndicesByKind(PoolIndexKind.Type) < 2) {
+ Log.debug("Cannot mutate, only one type, skipping...");
+ return false;
+ }
+
+ for (MInsn mInsn : mutatableCode.getInstructions()) {
+ if (mInsn.insn.info.opcode == Opcode.NEW_INSTANCE) {
+ return true;
+ }
+ }
+ Log.debug("No New Instance in method, skipping...");
+ return false;
+ }
+
+ @Override
+ protected Mutation generateMutation(MutatableCode mutatableCode) {
+ generateCachedNewInstanceInsns(mutatableCode);
+
+ int newInstanceIdxInCache = rng.nextInt(newInstanceCachedInsns.size());
+ MInsn newInstanceInsn = newInstanceCachedInsns.get(newInstanceIdxInCache);
+ int oldTypeIdx = (int) newInstanceInsn.insn.vregB;
+ int newTypeIdx = 0;
+ int totalPoolIndices = mutatableCode.program.getTotalPoolIndicesByKind(PoolIndexKind.Type);
+ if (totalPoolIndices < 2) {
+ Log.errorAndQuit("Less than two types present, quitting...");
+ }
+
+ while (newTypeIdx == oldTypeIdx) {
+ newTypeIdx = rng.nextInt(totalPoolIndices);
+ }
+
+ AssociatedMutation mutation = new AssociatedMutation();
+ mutation.setup(this.getClass(), mutatableCode);
+ mutation.newInstanceToChangeIdx = newInstanceIdxInCache;
+ mutation.newInstanceTypeIdx = newTypeIdx;
+ return mutation;
+ }
+
+ @Override
+ protected void applyMutation(Mutation uncastMutation) {
+ // Cast the Mutation to our AssociatedMutation, so we can access its fields.
+ AssociatedMutation mutation = (AssociatedMutation) uncastMutation;
+ MutatableCode mutatableCode = mutation.mutatableCode;
+
+ generateCachedNewInstanceInsns(mutatableCode);
+
+ MInsn newInstanceInsn = newInstanceCachedInsns.get(mutation.newInstanceToChangeIdx);
+
+ ContainsPoolIndex poolIndex = ((ContainsPoolIndex)newInstanceInsn.insn.info.format);
+
+ poolIndex.setPoolIndex(newInstanceInsn.insn, mutation.newInstanceTypeIdx);
+
+ Log.info("Changed the type of " + newInstanceInsn.toString() +
+ " to " + mutation.newInstanceTypeIdx);
+
+ int foundNewInstanceInsnIdx =
+ foundInsnIdx(mutatableCode, newInstanceCachedInsns.get(mutation.newInstanceToChangeIdx));
+
+ changeInvokeDirect(foundNewInstanceInsnIdx, mutation);
+
+ stats.incrementStat("Changed new instance.");
+
+ // Clear cache.
+ newInstanceCachedInsns = null;
+ }
+
+ /**
+ * Try to find the invoke-direct/ invoke-direct-range instruction that follows
+ * the new instance instruction and change the method ID of the instruction.
+ * @param foundInsnIdx
+ * @param uncastMutation
+ */
+ protected void changeInvokeDirect(int foundInsnIdx, Mutation uncastMutation) {
+ AssociatedMutation mutation = (AssociatedMutation) uncastMutation;
+ MutatableCode mutatableCode = mutation.mutatableCode;
+ if (foundInsnIdx == -1 ||
+ foundInsnIdx + 1 == mutatableCode.getInstructionCount()) {
+ return;
+ }
+
+ MInsn insn = mutatableCode.getInstructionAt(foundInsnIdx + 1);
+ if (isInvokeInst(insn)) {
+ ContainsPoolIndex poolIndex =((ContainsPoolIndex)insn.insn.info.format);
+ long oldMethodIdx = poolIndex.getPoolIndex(insn.insn);
+ String className = mutatableCode.program.getTypeString(mutation.newInstanceTypeIdx);
+ String methodName = mutatableCode.program.getMethodString((int) oldMethodIdx);
+ String shorty = mutatableCode.program.getMethodProto((int) oldMethodIdx);
+
+ // Matches the type of the invoke with the randomly changed type of the prior new-instance.
+ // This might create a lot of verification failures but still works many times.
+ // TODO: Work on generating a program which finds a valid type.
+ int methodId = mutatableCode.program.getNewItemCreator().
+ findOrCreateMethodId(className, methodName, shorty);
+
+ poolIndex.setPoolIndex(insn.insn, mutation.newInstanceTypeIdx);
+
+ insn.insn.vregB = methodId;
+
+ Log.info("Changed " + oldMethodIdx + " to " + methodId);
+ }
+ }
+
+ protected boolean isInvokeInst(MInsn mInsn) {
+ return (mInsn.insn.info.opcode == Opcode.INVOKE_DIRECT ||
+ mInsn.insn.info.opcode == Opcode.INVOKE_DIRECT_RANGE);
+ }
+
+ // Check if there is an new instance instruction, and if found, return the index.
+ // If not, return -1.
+ protected int foundInsnIdx(MutatableCode mutatableCode, MInsn newInstanceInsn) {
+ int i = 0;
+ for (MInsn mInsn : mutatableCode.getInstructions()) {
+ if (mInsn == newInstanceInsn) {
+ return i;
+ }
+ i++;
+ }
+ return -1;
+ }
+}
diff --git a/tools/libcore_gcstress_debug_failures.txt b/tools/libcore_gcstress_debug_failures.txt
index 66bc252..5806b61 100644
--- a/tools/libcore_gcstress_debug_failures.txt
+++ b/tools/libcore_gcstress_debug_failures.txt
@@ -8,9 +8,11 @@
description: "Timeouts on target with gcstress and debug.",
result: EXEC_FAILED,
modes: [device],
- names: ["libcore.icu.TransliteratorTest#testAll",
+ names: ["jsr166.CompletableFutureTest#testCompleteOnTimeout_completed",
+ "libcore.icu.TransliteratorTest#testAll",
"libcore.icu.RelativeDateTimeFormatterTest#test_bug25821045",
"libcore.java.lang.ref.ReferenceQueueTest#testRemoveWithDelayedResultAndTimeout",
+ "libcore.java.lang.ref.ReferenceQueueTest#testRemoveWithDelayedResultAndNoTimeout",
"libcore.java.util.TimeZoneTest#testSetDefaultDeadlock",
"org.apache.harmony.tests.java.util.TimerTest#testThrowingTaskKillsTimerThread"]
}
diff --git a/tools/run-jdwp-tests.sh b/tools/run-jdwp-tests.sh
index d2322bb..355646b 100755
--- a/tools/run-jdwp-tests.sh
+++ b/tools/run-jdwp-tests.sh
@@ -122,6 +122,9 @@
fi
done
+# Make sure the debuggee doesn't clean up what the debugger has generated.
+art_debugee="$art_debugee --no-clean"
+
# For the host:
#
# If, on the other hand, there is a variant set, use it to modify the art_debugee parameter to