diff options
24 files changed, 826 insertions, 79 deletions
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index 672018b355..41b19601b9 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -444,7 +444,7 @@ TEST_F(OatTest, OatHeaderSizeCheck) { EXPECT_EQ(72U, sizeof(OatHeader)); EXPECT_EQ(4U, sizeof(OatMethodOffsets)); EXPECT_EQ(20U, sizeof(OatQuickMethodHeader)); - EXPECT_EQ(133 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints)); + EXPECT_EQ(164 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints)); } TEST_F(OatTest, OatHeaderIsValid) { diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index 9364be35ff..b8540baca2 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -350,6 +350,16 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { // accessing the String's `value` field in String intrinsics. static uint32_t GetArrayDataOffset(HArrayGet* array_get); + // Return the entry point offset for ReadBarrierMarkRegX, where X is `reg`. + template <size_t pointer_size> + static int32_t GetReadBarrierMarkEntryPointsOffset(size_t reg) { + DCHECK_LT(reg, 32u); + // The ReadBarrierMarkRegX entry points are ordered by increasing + // register number in Thread::tls_Ptr_.quick_entrypoints. + return QUICK_ENTRYPOINT_OFFSET(pointer_size, pReadBarrierMarkReg00).Int32Value() + + pointer_size * reg; + } + void EmitParallelMoves(Location from1, Location to1, Primitive::Type type1, diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 5316d59bff..e9bc3b36dd 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -412,8 +412,8 @@ class ArraySetSlowPathARM : public SlowPathCode { // Slow path marking an object during a read barrier. class ReadBarrierMarkSlowPathARM : public SlowPathCode { public: - ReadBarrierMarkSlowPathARM(HInstruction* instruction, Location out, Location obj) - : SlowPathCode(instruction), out_(out), obj_(obj) { + ReadBarrierMarkSlowPathARM(HInstruction* instruction, Location obj) + : SlowPathCode(instruction), obj_(obj) { DCHECK(kEmitCompilerReadBarrier); } @@ -421,9 +421,9 @@ class ReadBarrierMarkSlowPathARM : public SlowPathCode { void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); - Register reg_out = out_.AsRegister<Register>(); + Register reg = obj_.AsRegister<Register>(); DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg)); DCHECK(instruction_->IsInstanceFieldGet() || instruction_->IsStaticFieldGet() || instruction_->IsArrayGet() || @@ -437,24 +437,44 @@ class ReadBarrierMarkSlowPathARM : public SlowPathCode { << instruction_->DebugName(); __ Bind(GetEntryLabel()); + // Save live registers before the runtime call, and in particular + // R0 (if it is live), as it is clobbered by functions + // art_quick_read_barrier_mark_regX. SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConvention calling_convention; CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_); - arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark), + DCHECK_NE(reg, SP); + DCHECK_NE(reg, LR); + DCHECK_NE(reg, PC); + DCHECK(0 <= reg && reg < kNumberOfCoreRegisters) << reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in R0): + // + // R0 <- obj + // R0 <- ReadBarrierMark(R0) + // obj <- R0 + // + // we just use rX (the register holding `obj`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmWordSize>(reg); + // TODO: Do not emit a stack map for this runtime call. + arm_codegen->InvokeRuntime(entry_point_offset, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>(); - arm_codegen->Move32(out_, Location::RegisterLocation(R0)); RestoreLiveRegisters(codegen, locations); __ b(GetExitLabel()); } private: - const Location out_; const Location obj_; DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); @@ -6175,7 +6195,7 @@ void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruct // Slow path used to mark the GC root `root`. SlowPathCode* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, root, root); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, root); codegen_->AddSlowPath(slow_path); // IP = Thread::Current()->GetIsGcMarking() @@ -6324,7 +6344,7 @@ void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* i // Slow path used to mark the object `ref` when it is gray. SlowPathCode* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref, ref); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); AddSlowPath(slow_path); // if (rb_state == ReadBarrier::gray_ptr_) diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index fc2c2c34aa..e480f12f26 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -580,8 +580,8 @@ void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { // Slow path marking an object during a read barrier. class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 { public: - ReadBarrierMarkSlowPathARM64(HInstruction* instruction, Location out, Location obj) - : SlowPathCodeARM64(instruction), out_(out), obj_(obj) { + ReadBarrierMarkSlowPathARM64(HInstruction* instruction, Location obj) + : SlowPathCodeARM64(instruction), obj_(obj) { DCHECK(kEmitCompilerReadBarrier); } @@ -589,9 +589,8 @@ class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 { void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); - Primitive::Type type = Primitive::kPrimNot; DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg())); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(obj_.reg())); DCHECK(instruction_->IsInstanceFieldGet() || instruction_->IsStaticFieldGet() || instruction_->IsArrayGet() || @@ -605,24 +604,44 @@ class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 { << instruction_->DebugName(); __ Bind(GetEntryLabel()); + // Save live registers before the runtime call, and in particular + // W0 (if it is live), as it is clobbered by functions + // art_quick_read_barrier_mark_regX. SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConvention calling_convention; CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); - arm64_codegen->MoveLocation(LocationFrom(calling_convention.GetRegisterAt(0)), obj_, type); - arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark), + DCHECK_NE(obj_.reg(), LR); + DCHECK_NE(obj_.reg(), WSP); + DCHECK_NE(obj_.reg(), WZR); + DCHECK(0 <= obj_.reg() && obj_.reg() < kNumberOfWRegisters) << obj_.reg(); + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in W0): + // + // W0 <- obj + // W0 <- ReadBarrierMark(W0) + // obj <- W0 + // + // we just use rX (the register holding `obj`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64WordSize>(obj_.reg()); + // TODO: Do not emit a stack map for this runtime call. + arm64_codegen->InvokeRuntime(entry_point_offset, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>(); - arm64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type); RestoreLiveRegisters(codegen, locations); __ B(GetExitLabel()); } private: - const Location out_; const Location obj_; DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM64); @@ -5059,7 +5078,7 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instru // Slow path used to mark the GC root `root`. SlowPathCodeARM64* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root, root); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root); codegen_->AddSlowPath(slow_path); MacroAssembler* masm = GetVIXLAssembler(); @@ -5267,7 +5286,7 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* // Slow path used to mark the object `ref` when it is gray. SlowPathCodeARM64* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, ref, ref); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, ref); AddSlowPath(slow_path); // if (rb_state == ReadBarrier::gray_ptr_) diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 1261619536..b33cabb2b9 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -430,8 +430,8 @@ class ArraySetSlowPathX86 : public SlowPathCode { // Slow path marking an object during a read barrier. class ReadBarrierMarkSlowPathX86 : public SlowPathCode { public: - ReadBarrierMarkSlowPathX86(HInstruction* instruction, Location out, Location obj) - : SlowPathCode(instruction), out_(out), obj_(obj) { + ReadBarrierMarkSlowPathX86(HInstruction* instruction, Location obj) + : SlowPathCode(instruction), obj_(obj) { DCHECK(kEmitCompilerReadBarrier); } @@ -439,9 +439,9 @@ class ReadBarrierMarkSlowPathX86 : public SlowPathCode { void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); - Register reg_out = out_.AsRegister<Register>(); + Register reg = obj_.AsRegister<Register>(); DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg)); DCHECK(instruction_->IsInstanceFieldGet() || instruction_->IsStaticFieldGet() || instruction_->IsArrayGet() || @@ -455,24 +455,42 @@ class ReadBarrierMarkSlowPathX86 : public SlowPathCode { << instruction_->DebugName(); __ Bind(GetEntryLabel()); + // Save live registers before the runtime call, and in particular + // EAX (if it is live), as it is clobbered by functions + // art_quick_read_barrier_mark_regX. SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConvention calling_convention; CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen); - x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_); - x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark), + DCHECK_NE(reg, ESP); + DCHECK(0 <= reg && reg < kNumberOfCpuRegisters) << reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in EAX): + // + // EAX <- obj + // EAX <- ReadBarrierMark(EAX) + // obj <- EAX + // + // we just use rX (the register holding `obj`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86WordSize>(reg); + // TODO: Do not emit a stack map for this runtime call. + x86_codegen->InvokeRuntime(entry_point_offset, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>(); - x86_codegen->Move32(out_, Location::RegisterLocation(EAX)); RestoreLiveRegisters(codegen, locations); __ jmp(GetExitLabel()); } private: - const Location out_; const Location obj_; DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathX86); @@ -6926,7 +6944,7 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruct // Slow path used to mark the GC root `root`. SlowPathCode* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, root, root); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, root); codegen_->AddSlowPath(slow_path); __ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86WordSize>().Int32Value()), @@ -7056,7 +7074,7 @@ void CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* i // Slow path used to mark the object `ref` when it is gray. SlowPathCode* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, ref, ref); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, ref); AddSlowPath(slow_path); // if (rb_state == ReadBarrier::gray_ptr_) diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 5e30203b38..a524057359 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -451,8 +451,8 @@ class ArraySetSlowPathX86_64 : public SlowPathCode { // Slow path marking an object during a read barrier. class ReadBarrierMarkSlowPathX86_64 : public SlowPathCode { public: - ReadBarrierMarkSlowPathX86_64(HInstruction* instruction, Location out, Location obj) - : SlowPathCode(instruction), out_(out), obj_(obj) { + ReadBarrierMarkSlowPathX86_64(HInstruction* instruction, Location obj) + : SlowPathCode(instruction), obj_(obj) { DCHECK(kEmitCompilerReadBarrier); } @@ -460,9 +460,9 @@ class ReadBarrierMarkSlowPathX86_64 : public SlowPathCode { void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); - Register reg_out = out_.AsRegister<Register>(); + Register reg = obj_.AsRegister<Register>(); DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg)); DCHECK(instruction_->IsInstanceFieldGet() || instruction_->IsStaticFieldGet() || instruction_->IsArrayGet() || @@ -476,24 +476,42 @@ class ReadBarrierMarkSlowPathX86_64 : public SlowPathCode { << instruction_->DebugName(); __ Bind(GetEntryLabel()); + // Save live registers before the runtime call, and in particular + // RDI and/or RAX (if they are live), as they are clobbered by + // functions art_quick_read_barrier_mark_regX. SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConvention calling_convention; CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen); - x86_64_codegen->Move(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_); - x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark), - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>(); - x86_64_codegen->Move(out_, Location::RegisterLocation(RAX)); + DCHECK_NE(reg, RSP); + DCHECK(0 <= reg && reg < kNumberOfCpuRegisters) << reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in R0): + // + // RDI <- obj + // RAX <- ReadBarrierMark(RDI) + // obj <- RAX + // + // we just use rX (the register holding `obj`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64WordSize>(reg); + // TODO: Do not emit a stack map for this runtime call. + x86_64_codegen->InvokeRuntime(entry_point_offset, + instruction_, + instruction_->GetDexPc(), + this); RestoreLiveRegisters(codegen, locations); __ jmp(GetExitLabel()); } private: - const Location out_; const Location obj_; DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathX86_64); @@ -6378,7 +6396,7 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instr // Slow path used to mark the GC root `root`. SlowPathCode* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86_64(instruction, root, root); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86_64(instruction, root); codegen_->AddSlowPath(slow_path); __ gs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86_64WordSize>().Int32Value(), @@ -6509,7 +6527,7 @@ void CodeGeneratorX86_64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction // Slow path used to mark the object `ref` when it is gray. SlowPathCode* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86_64(instruction, ref, ref); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86_64(instruction, ref); AddSlowPath(slow_path); // if (rb_state == ReadBarrier::gray_ptr_) diff --git a/runtime/arch/arm/entrypoints_init_arm.cc b/runtime/arch/arm/entrypoints_init_arm.cc index 4c68862c7f..8f18ff3204 100644 --- a/runtime/arch/arm/entrypoints_init_arm.cc +++ b/runtime/arch/arm/entrypoints_init_arm.cc @@ -30,6 +30,22 @@ namespace art { extern "C" uint32_t artIsAssignableFromCode(const mirror::Class* klass, const mirror::Class* ref_class); +// Read barrier entrypoints. +// art_quick_read_barrier_mark_regX uses an non-standard calling +// convention: it expects its input in register X and returns its +// result in that same register. +extern "C" mirror::Object* art_quick_read_barrier_mark_reg01(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg02(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg03(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg04(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg05(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg06(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg07(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg08(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg09(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg10(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg11(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg12(mirror::Object*); // Used by soft float. // Single-precision FP arithmetics. @@ -103,7 +119,39 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - qpoints->pReadBarrierMark = artReadBarrierMark; + qpoints->pReadBarrierMarkReg00 = artReadBarrierMark; + qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01; + qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02; + qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03; + qpoints->pReadBarrierMarkReg04 = art_quick_read_barrier_mark_reg04; + qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05; + qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06; + qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07; + qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08; + qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09; + qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10; + qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11; + qpoints->pReadBarrierMarkReg12 = art_quick_read_barrier_mark_reg12; + qpoints->pReadBarrierMarkReg13 = nullptr; // Cannot use register 13 (SP) to pass arguments. + qpoints->pReadBarrierMarkReg14 = nullptr; // Cannot use register 14 (LR) to pass arguments. + qpoints->pReadBarrierMarkReg15 = nullptr; // Cannot use register 15 (PC) to pass arguments. + // ARM has only 16 core registers. + qpoints->pReadBarrierMarkReg16 = nullptr; + qpoints->pReadBarrierMarkReg17 = nullptr; + qpoints->pReadBarrierMarkReg18 = nullptr; + qpoints->pReadBarrierMarkReg19 = nullptr; + qpoints->pReadBarrierMarkReg20 = nullptr; + qpoints->pReadBarrierMarkReg21 = nullptr; + qpoints->pReadBarrierMarkReg22 = nullptr; + qpoints->pReadBarrierMarkReg23 = nullptr; + qpoints->pReadBarrierMarkReg24 = nullptr; + qpoints->pReadBarrierMarkReg25 = nullptr; + qpoints->pReadBarrierMarkReg26 = nullptr; + qpoints->pReadBarrierMarkReg27 = nullptr; + qpoints->pReadBarrierMarkReg28 = nullptr; + qpoints->pReadBarrierMarkReg29 = nullptr; + qpoints->pReadBarrierMarkReg30 = nullptr; + qpoints->pReadBarrierMarkReg31 = nullptr; qpoints->pReadBarrierSlow = artReadBarrierSlow; qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow; } diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index d940164c2c..82d64b90d7 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -1743,3 +1743,43 @@ ENTRY art_quick_l2f .cfi_adjust_cfa_offset -4 pop {pc} END art_quick_l2f + + /* + * Create a function `name` calling the ReadBarrier::Mark routine, + * getting its argument and returning its result through register + * `reg`, thus following a non-standard runtime calling convention: + * - `reg` is used to pass the (sole) argument of this function + * (instead of R0); + * - `reg` is used to return the result of this function (instead of R0); + * - R0 is treated like a normal (non-argument) caller-save register; + * - everything else is the same as in the standard runtime calling + * convention (e.g. same callee-save registers). + */ +.macro READ_BARRIER_MARK_REG name, reg +ENTRY \name + push {lr} @ save return address + .cfi_adjust_cfa_offset 4 + .cfi_rel_offset lr, 0 + sub sp, #4 @ push padding (native calling convention 8-byte alignment) + .cfi_adjust_cfa_offset 4 + mov r0, \reg @ pass arg1 - obj from `reg` + bl artReadBarrierMark @ artReadBarrierMark(obj) + mov \reg, r0 @ return result into `reg` + add sp, #4 @ pop padding + .cfi_adjust_cfa_offset -4 + pop {pc} @ return +END \name +.endm + +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, r1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, r2 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, r3 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, r4 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, r5 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, r6 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, r7 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, r8 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, r9 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, r10 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, r11 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, r12 diff --git a/runtime/arch/arm64/entrypoints_init_arm64.cc b/runtime/arch/arm64/entrypoints_init_arm64.cc index bf0f6470d1..c3188b6aad 100644 --- a/runtime/arch/arm64/entrypoints_init_arm64.cc +++ b/runtime/arch/arm64/entrypoints_init_arm64.cc @@ -30,6 +30,42 @@ namespace art { extern "C" uint32_t artIsAssignableFromCode(const mirror::Class* klass, const mirror::Class* ref_class); +// Read barrier entrypoints. +// art_quick_read_barrier_mark_regX uses an non-standard calling +// convention: it expects its input in register X and returns its +// result in that same register. +extern "C" mirror::Object* art_quick_read_barrier_mark_reg01(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg02(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg03(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg04(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg05(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg06(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg07(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg08(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg09(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg10(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg11(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg12(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg12(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg13(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg14(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg15(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg16(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg17(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg18(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg19(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg20(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg21(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg22(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg22(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg23(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg24(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg25(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg26(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg27(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg28(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg29(mirror::Object*); + void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { DefaultInitEntryPoints(jpoints, qpoints); @@ -86,7 +122,38 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - qpoints->pReadBarrierMark = artReadBarrierMark; + qpoints->pReadBarrierMarkReg00 = artReadBarrierMark; + qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01; + qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02; + qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03; + qpoints->pReadBarrierMarkReg04 = art_quick_read_barrier_mark_reg04; + qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05; + qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06; + qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07; + qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08; + qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09; + qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10; + qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11; + qpoints->pReadBarrierMarkReg12 = art_quick_read_barrier_mark_reg12; + qpoints->pReadBarrierMarkReg13 = art_quick_read_barrier_mark_reg13; + qpoints->pReadBarrierMarkReg14 = art_quick_read_barrier_mark_reg14; + qpoints->pReadBarrierMarkReg15 = art_quick_read_barrier_mark_reg15; + qpoints->pReadBarrierMarkReg16 = art_quick_read_barrier_mark_reg16; + qpoints->pReadBarrierMarkReg17 = art_quick_read_barrier_mark_reg17; + qpoints->pReadBarrierMarkReg18 = art_quick_read_barrier_mark_reg18; + qpoints->pReadBarrierMarkReg19 = art_quick_read_barrier_mark_reg19; + qpoints->pReadBarrierMarkReg20 = art_quick_read_barrier_mark_reg20; + qpoints->pReadBarrierMarkReg21 = art_quick_read_barrier_mark_reg21; + qpoints->pReadBarrierMarkReg22 = art_quick_read_barrier_mark_reg22; + qpoints->pReadBarrierMarkReg23 = art_quick_read_barrier_mark_reg23; + qpoints->pReadBarrierMarkReg24 = art_quick_read_barrier_mark_reg24; + qpoints->pReadBarrierMarkReg25 = art_quick_read_barrier_mark_reg25; + qpoints->pReadBarrierMarkReg26 = art_quick_read_barrier_mark_reg26; + qpoints->pReadBarrierMarkReg27 = art_quick_read_barrier_mark_reg27; + qpoints->pReadBarrierMarkReg28 = art_quick_read_barrier_mark_reg28; + qpoints->pReadBarrierMarkReg29 = art_quick_read_barrier_mark_reg29; + qpoints->pReadBarrierMarkReg30 = nullptr; // Cannot use register 30 (LR) to pass arguments. + qpoints->pReadBarrierMarkReg31 = nullptr; // Cannot use register 31 (SP/XZR) to pass arguments. qpoints->pReadBarrierSlow = artReadBarrierSlow; qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow; }; diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index 10ee63f74f..e9ad1f4080 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -2214,3 +2214,59 @@ ENTRY art_quick_indexof asr x0, x0, #1 ret END art_quick_indexof + + /* + * Create a function `name` calling the ReadBarrier::Mark routine, + * getting its argument and returning its result through register + * `reg`, thus following a non-standard runtime calling convention: + * - `reg` is used to pass the (sole) argument of this function + * (instead of W0); + * - `reg` is used to return the result of this function (instead of W0); + * - W0 is treated like a normal (non-argument) caller-save register; + * - everything else is the same as in the standard runtime calling + * convention (e.g. same callee-save registers). + */ +.macro READ_BARRIER_MARK_REG name, reg +ENTRY \name + str xLR, [sp, #-16]! // Save return address and add padding (16B align stack). + .cfi_adjust_cfa_offset 16 + .cfi_rel_offset x30, 0 + mov w0, \reg // Pass arg1 - obj from `reg` + bl artReadBarrierMark // artReadBarrierMark(obj) + mov \reg, w0 // Return result into `reg` + ldr xLR, [sp], #16 // Restore return address and remove padding. + .cfi_restore x30 + .cfi_adjust_cfa_offset -16 + ret +END \name +.endm + +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, w1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, w2 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, w3 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, w4 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, w5 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, w6 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, w7 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, w8 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, w9 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, w10 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, w11 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, w12 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, w13 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, w14 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, w15 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg16, w16 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, w17 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, w18 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, w19 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, w20 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, w21 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, w22 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg23, w23 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg24, w24 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg25, w25 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg26, w26 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg27, w27 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg28, w28 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, w29 diff --git a/runtime/arch/mips/entrypoints_direct_mips.h b/runtime/arch/mips/entrypoints_direct_mips.h index 5b74d620ac..937cd1ec57 100644 --- a/runtime/arch/mips/entrypoints_direct_mips.h +++ b/runtime/arch/mips/entrypoints_direct_mips.h @@ -46,7 +46,6 @@ static constexpr bool IsDirectEntrypoint(QuickEntrypointEnum entrypoint) { entrypoint == kQuickCmplDouble || entrypoint == kQuickCmplFloat || entrypoint == kQuickReadBarrierJni || - entrypoint == kQuickReadBarrierMark || entrypoint == kQuickReadBarrierSlow || entrypoint == kQuickReadBarrierForRootSlow; } diff --git a/runtime/arch/mips/entrypoints_init_mips.cc b/runtime/arch/mips/entrypoints_init_mips.cc index 6697a8d05c..e3cc0e0b67 100644 --- a/runtime/arch/mips/entrypoints_init_mips.cc +++ b/runtime/arch/mips/entrypoints_init_mips.cc @@ -284,8 +284,104 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; static_assert(IsDirectEntrypoint(kQuickReadBarrierJni), "Direct C stub not marked direct."); - qpoints->pReadBarrierMark = artReadBarrierMark; - static_assert(IsDirectEntrypoint(kQuickReadBarrierMark), "Direct C stub not marked direct."); + // Read barriers (and these entry points in particular) are not + // supported in the compiler on MIPS32. + qpoints->pReadBarrierMarkReg00 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg00), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg01 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg01), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg02 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg02), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg03 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg03), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg04 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg04), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg05 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg05), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg06 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg06), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg07 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg07), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg08 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg08), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg09 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg09), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg10 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg10), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg11 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg11), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg12 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg12), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg13 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg13), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg14 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg14), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg15 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg15), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg16 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg16), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg17 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg17), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg18 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg18), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg19 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg19), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg20 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg20), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg21 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg21), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg22 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg22), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg23 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg23), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg24 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg24), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg25 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg25), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg26 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg26), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg27 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg27), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg28 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg28), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg29 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg29), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg30 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg30), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg31 = nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg31), + "Non-direct C stub marked direct."); qpoints->pReadBarrierSlow = artReadBarrierSlow; static_assert(IsDirectEntrypoint(kQuickReadBarrierSlow), "Direct C stub not marked direct."); qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow; diff --git a/runtime/arch/mips64/entrypoints_init_mips64.cc b/runtime/arch/mips64/entrypoints_init_mips64.cc index 030c12707e..b19aa01712 100644 --- a/runtime/arch/mips64/entrypoints_init_mips64.cc +++ b/runtime/arch/mips64/entrypoints_init_mips64.cc @@ -97,7 +97,40 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - qpoints->pReadBarrierMark = artReadBarrierMark; + // Read barriers (and these entry points in particular) are not + // supported in the compiler on MIPS64. + qpoints->pReadBarrierMarkReg00 = nullptr; + qpoints->pReadBarrierMarkReg01 = nullptr; + qpoints->pReadBarrierMarkReg02 = nullptr; + qpoints->pReadBarrierMarkReg03 = nullptr; + qpoints->pReadBarrierMarkReg04 = nullptr; + qpoints->pReadBarrierMarkReg05 = nullptr; + qpoints->pReadBarrierMarkReg06 = nullptr; + qpoints->pReadBarrierMarkReg07 = nullptr; + qpoints->pReadBarrierMarkReg08 = nullptr; + qpoints->pReadBarrierMarkReg09 = nullptr; + qpoints->pReadBarrierMarkReg10 = nullptr; + qpoints->pReadBarrierMarkReg11 = nullptr; + qpoints->pReadBarrierMarkReg12 = nullptr; + qpoints->pReadBarrierMarkReg13 = nullptr; + qpoints->pReadBarrierMarkReg14 = nullptr; + qpoints->pReadBarrierMarkReg15 = nullptr; + qpoints->pReadBarrierMarkReg16 = nullptr; + qpoints->pReadBarrierMarkReg17 = nullptr; + qpoints->pReadBarrierMarkReg18 = nullptr; + qpoints->pReadBarrierMarkReg19 = nullptr; + qpoints->pReadBarrierMarkReg20 = nullptr; + qpoints->pReadBarrierMarkReg21 = nullptr; + qpoints->pReadBarrierMarkReg22 = nullptr; + qpoints->pReadBarrierMarkReg23 = nullptr; + qpoints->pReadBarrierMarkReg24 = nullptr; + qpoints->pReadBarrierMarkReg25 = nullptr; + qpoints->pReadBarrierMarkReg26 = nullptr; + qpoints->pReadBarrierMarkReg27 = nullptr; + qpoints->pReadBarrierMarkReg28 = nullptr; + qpoints->pReadBarrierMarkReg29 = nullptr; + qpoints->pReadBarrierMarkReg30 = nullptr; + qpoints->pReadBarrierMarkReg31 = nullptr; qpoints->pReadBarrierSlow = artReadBarrierSlow; qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow; }; diff --git a/runtime/arch/stub_test.cc b/runtime/arch/stub_test.cc index a7d6d6f793..09af3731d3 100644 --- a/runtime/arch/stub_test.cc +++ b/runtime/arch/stub_test.cc @@ -2153,6 +2153,8 @@ TEST_F(StubTest, StringIndexOf) { #endif } +// TODO: Exercise the ReadBarrierMarkRegX entry points. + TEST_F(StubTest, ReadBarrier) { #if defined(ART_USE_READ_BARRIER) && (defined(__i386__) || defined(__arm__) || \ defined(__aarch64__) || defined(__mips__) || (defined(__x86_64__) && !defined(__APPLE__))) diff --git a/runtime/arch/x86/entrypoints_init_x86.cc b/runtime/arch/x86/entrypoints_init_x86.cc index 15a857146b..69c939e4cb 100644 --- a/runtime/arch/x86/entrypoints_init_x86.cc +++ b/runtime/arch/x86/entrypoints_init_x86.cc @@ -29,7 +29,16 @@ extern "C" uint32_t art_quick_is_assignable(const mirror::Class* klass, const mirror::Class* ref_class); // Read barrier entrypoints. -extern "C" mirror::Object* art_quick_read_barrier_mark(mirror::Object*); +// art_quick_read_barrier_mark_regX uses an non-standard calling +// convention: it expects its input in register X and returns its +// result in that same register. +extern "C" mirror::Object* art_quick_read_barrier_mark_reg00(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg01(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg02(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg03(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg05(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg06(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg07(mirror::Object*); extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t); extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*); @@ -76,7 +85,39 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - qpoints->pReadBarrierMark = art_quick_read_barrier_mark; + qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00; + qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01; + qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02; + qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03; + qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (ESP) to pass arguments. + qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05; + qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06; + qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07; + // x86 has only 8 core registers. + qpoints->pReadBarrierMarkReg08 = nullptr; + qpoints->pReadBarrierMarkReg09 = nullptr; + qpoints->pReadBarrierMarkReg10 = nullptr; + qpoints->pReadBarrierMarkReg11 = nullptr; + qpoints->pReadBarrierMarkReg12 = nullptr; + qpoints->pReadBarrierMarkReg13 = nullptr; + qpoints->pReadBarrierMarkReg14 = nullptr; + qpoints->pReadBarrierMarkReg15 = nullptr; + qpoints->pReadBarrierMarkReg16 = nullptr; + qpoints->pReadBarrierMarkReg17 = nullptr; + qpoints->pReadBarrierMarkReg18 = nullptr; + qpoints->pReadBarrierMarkReg19 = nullptr; + qpoints->pReadBarrierMarkReg20 = nullptr; + qpoints->pReadBarrierMarkReg21 = nullptr; + qpoints->pReadBarrierMarkReg22 = nullptr; + qpoints->pReadBarrierMarkReg23 = nullptr; + qpoints->pReadBarrierMarkReg24 = nullptr; + qpoints->pReadBarrierMarkReg25 = nullptr; + qpoints->pReadBarrierMarkReg26 = nullptr; + qpoints->pReadBarrierMarkReg27 = nullptr; + qpoints->pReadBarrierMarkReg28 = nullptr; + qpoints->pReadBarrierMarkReg29 = nullptr; + qpoints->pReadBarrierMarkReg30 = nullptr; + qpoints->pReadBarrierMarkReg31 = nullptr; qpoints->pReadBarrierSlow = art_quick_read_barrier_slow; qpoints->pReadBarrierForRootSlow = art_quick_read_barrier_for_root_slow; }; diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S index 6234f0f73a..e75fecba4b 100644 --- a/runtime/arch/x86/quick_entrypoints_x86.S +++ b/runtime/arch/x86/quick_entrypoints_x86.S @@ -1908,7 +1908,10 @@ DEFINE_FUNCTION art_nested_signal_return UNREACHABLE END_FUNCTION art_nested_signal_return -DEFINE_FUNCTION art_quick_read_barrier_mark +// Call the ReadBarrierMark entry point, getting input and returning +// result through EAX (register 0), following the standard runtime +// calling convention. +DEFINE_FUNCTION art_quick_read_barrier_mark_reg00 subl LITERAL(8), %esp // alignment padding CFI_ADJUST_CFA_OFFSET(8) PUSH eax // pass arg1 - obj @@ -1916,7 +1919,38 @@ DEFINE_FUNCTION art_quick_read_barrier_mark addl LITERAL(12), %esp // pop argument and remove padding CFI_ADJUST_CFA_OFFSET(-12) ret -END_FUNCTION art_quick_read_barrier_mark +END_FUNCTION art_quick_read_barrier_mark_reg00 + +// Create a function `name` calling the ReadBarrier::Mark routine, +// getting its argument and returning its result through register +// `reg`, thus following a non-standard runtime calling convention: +// - `reg` is used to pass the (sole) argument of this function +// (instead of EAX); +// - `reg` is used to return the result of this function (instead of EAX); +// - EAX is treated like a normal (non-argument) caller-save register; +// - everything else is the same as in the standard runtime calling +// convention (e.g. same callee-save registers). +MACRO2(READ_BARRIER_MARK_REG, name, reg) + DEFINE_FUNCTION VAR(name) + subl LITERAL(8), %esp // alignment padding + CFI_ADJUST_CFA_OFFSET(8) + PUSH RAW_VAR(reg) // pass arg1 - obj from `reg` + call SYMBOL(artReadBarrierMark) // artReadBarrierMark(obj) + movl %eax, REG_VAR(reg) // return result into `reg` + addl LITERAL(12), %esp // pop argument and remove padding + CFI_ADJUST_CFA_OFFSET(-12) + ret + END_FUNCTION VAR(name) +END_MACRO + +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, ecx +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, edx +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, ebx +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, ebp +// Note: There is no art_quick_read_barrier_mark_reg04, as register 4 (ESP) +// cannot be used to pass arguments. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, esi +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, edi DEFINE_FUNCTION art_quick_read_barrier_slow PUSH edx // pass arg3 - offset diff --git a/runtime/arch/x86_64/entrypoints_init_x86_64.cc b/runtime/arch/x86_64/entrypoints_init_x86_64.cc index bd6df700d0..2bea3dbf61 100644 --- a/runtime/arch/x86_64/entrypoints_init_x86_64.cc +++ b/runtime/arch/x86_64/entrypoints_init_x86_64.cc @@ -32,7 +32,24 @@ extern "C" uint32_t art_quick_assignable_from_code(const mirror::Class* klass, const mirror::Class* ref_class); // Read barrier entrypoints. -extern "C" mirror::Object* art_quick_read_barrier_mark(mirror::Object*); +// art_quick_read_barrier_mark_regX uses an non-standard calling +// convention: it expects its input in register X and returns its +// result in that same register. +extern "C" mirror::Object* art_quick_read_barrier_mark_reg00(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg01(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg02(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg03(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg05(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg06(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg07(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg08(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg09(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg10(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg11(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg12(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg13(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg14(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg15(mirror::Object*); extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t); extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*); @@ -82,7 +99,39 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - qpoints->pReadBarrierMark = art_quick_read_barrier_mark; + qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00; + qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01; + qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02; + qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03; + qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (RSP) to pass arguments. + qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05; + qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06; + qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07; + qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08; + qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09; + qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10; + qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11; + qpoints->pReadBarrierMarkReg12 = art_quick_read_barrier_mark_reg12; + qpoints->pReadBarrierMarkReg13 = art_quick_read_barrier_mark_reg13; + qpoints->pReadBarrierMarkReg14 = art_quick_read_barrier_mark_reg14; + qpoints->pReadBarrierMarkReg15 = art_quick_read_barrier_mark_reg15; + // x86-64 has only 16 core registers. + qpoints->pReadBarrierMarkReg16 = nullptr; + qpoints->pReadBarrierMarkReg17 = nullptr; + qpoints->pReadBarrierMarkReg18 = nullptr; + qpoints->pReadBarrierMarkReg19 = nullptr; + qpoints->pReadBarrierMarkReg20 = nullptr; + qpoints->pReadBarrierMarkReg21 = nullptr; + qpoints->pReadBarrierMarkReg22 = nullptr; + qpoints->pReadBarrierMarkReg23 = nullptr; + qpoints->pReadBarrierMarkReg24 = nullptr; + qpoints->pReadBarrierMarkReg25 = nullptr; + qpoints->pReadBarrierMarkReg26 = nullptr; + qpoints->pReadBarrierMarkReg27 = nullptr; + qpoints->pReadBarrierMarkReg28 = nullptr; + qpoints->pReadBarrierMarkReg29 = nullptr; + qpoints->pReadBarrierMarkReg30 = nullptr; + qpoints->pReadBarrierMarkReg31 = nullptr; qpoints->pReadBarrierSlow = art_quick_read_barrier_slow; qpoints->pReadBarrierForRootSlow = art_quick_read_barrier_for_root_slow; #endif // __APPLE__ diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S index e777e6cfb2..496e6a8b4a 100644 --- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S +++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S @@ -1815,16 +1815,89 @@ DEFINE_FUNCTION art_nested_signal_return UNREACHABLE END_FUNCTION art_nested_signal_return -DEFINE_FUNCTION art_quick_read_barrier_mark +// Call the ReadBarrier::Mark routine, getting argument and returning +// result through RAX (register 0), thus following a non-standard +// runtime calling convention: +// - RAX is used to pass the (sole) argument of this function (instead +// of RDI); +// - RDI is treated like a normal (non-argument) caller-save register; +// - everything else is the same as in the standard runtime calling +// convention; in particular, RAX is still used to return the result +// of this function. +DEFINE_FUNCTION art_quick_read_barrier_mark_reg00 SETUP_FP_CALLEE_SAVE_FRAME subq LITERAL(8), %rsp // Alignment padding. CFI_ADJUST_CFA_OFFSET(8) + movq %rax, %rdi // Pass arg1 - obj from RAX. call SYMBOL(artReadBarrierMark) // artReadBarrierMark(obj) - addq LITERAL(8), %rsp + addq LITERAL(8), %rsp // Remove padding. CFI_ADJUST_CFA_OFFSET(-8) RESTORE_FP_CALLEE_SAVE_FRAME ret -END_FUNCTION art_quick_read_barrier_slow +END_FUNCTION art_quick_read_barrier_mark_reg00 + +// Call the ReadBarrier::Mark routine, getting argument and returning +// result through RDI (register 7), thus following a non-standard +// runtime calling convention: +// - RDI is used to return the result of this function (instead of RAX); +// - RAX is treated like a normal (non-result) caller-save register; +// - everything else is the same as in the standard runtime calling +// convention; in particular, RDI is still used to pass the (sole) +// argument of this function. +DEFINE_FUNCTION art_quick_read_barrier_mark_reg07 + SETUP_FP_CALLEE_SAVE_FRAME + subq LITERAL(8), %rsp // Alignment padding. + CFI_ADJUST_CFA_OFFSET(8) + call SYMBOL(artReadBarrierMark) // artReadBarrierMark(obj) + movq %rax, %rdi // Return result into RDI. + addq LITERAL(8), %rsp // Remove padding. + CFI_ADJUST_CFA_OFFSET(-8) + RESTORE_FP_CALLEE_SAVE_FRAME + ret +END_FUNCTION art_quick_read_barrier_mark_reg07 + +// Create a function `name` calling the ReadBarrier::Mark routine, +// getting its argument and returning its result through register +// `reg`, thus following a non-standard runtime calling convention: +// - `reg` is used to pass the (sole) argument of this function (instead +// of RDI); +// - `reg` is used to return the result of this function (instead of RAX); +// - RDI is treated like a normal (non-argument) caller-save register; +// - RAX is treated like a normal (non-result) caller-save register; +// - everything else is the same as in the standard runtime calling +// convention (e.g. same callee-save registers). +MACRO2(READ_BARRIER_MARK_REG, name, reg) + DEFINE_FUNCTION VAR(name) + SETUP_FP_CALLEE_SAVE_FRAME + subq LITERAL(8), %rsp // Alignment padding. + CFI_ADJUST_CFA_OFFSET(8) + movq REG_VAR(reg), %rdi // Pass arg1 - obj from `reg`. + call SYMBOL(artReadBarrierMark) // artReadBarrierMark(obj) + movq %rax, REG_VAR(reg) // Return result into `reg`. + addq LITERAL(8), %rsp // Remove padding. + CFI_ADJUST_CFA_OFFSET(-8) + RESTORE_FP_CALLEE_SAVE_FRAME + ret + END_FUNCTION VAR(name) +END_MACRO + +// Note: art_quick_read_barrier_mark_reg00 is implemented above. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, rcx +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, rdx +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, rbx +// Note: There is no art_quick_read_barrier_mark_reg04, as register 4 (RSP) +// cannot be used to pass arguments. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, rbp +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, rsi +// Note: art_quick_read_barrier_mark_reg07 is implemented above. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, r8 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, r9 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, r10 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, r11 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, r12 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, r13 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, r14 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, r15 DEFINE_FUNCTION art_quick_read_barrier_slow SETUP_FP_CALLEE_SAVE_FRAME diff --git a/runtime/asm_support.h b/runtime/asm_support.h index da68923cdd..50a786f1db 100644 --- a/runtime/asm_support.h +++ b/runtime/asm_support.h @@ -127,20 +127,20 @@ ADD_TEST_EQ(THREAD_TOP_QUICK_FRAME_OFFSET, ADD_TEST_EQ(THREAD_SELF_OFFSET, art::Thread::SelfOffset<__SIZEOF_POINTER__>().Int32Value()) +// Offset of field Thread::tlsPtr_.thread_local_objects. +#define THREAD_LOCAL_OBJECTS_OFFSET (THREAD_CARD_TABLE_OFFSET + 199 * __SIZEOF_POINTER__) +ADD_TEST_EQ(THREAD_LOCAL_OBJECTS_OFFSET, + art::Thread::ThreadLocalObjectsOffset<__SIZEOF_POINTER__>().Int32Value()) // Offset of field Thread::tlsPtr_.thread_local_pos. -#define THREAD_LOCAL_POS_OFFSET (THREAD_CARD_TABLE_OFFSET + 168 * __SIZEOF_POINTER__) +#define THREAD_LOCAL_POS_OFFSET (THREAD_LOCAL_OBJECTS_OFFSET + __SIZEOF_SIZE_T__) ADD_TEST_EQ(THREAD_LOCAL_POS_OFFSET, art::Thread::ThreadLocalPosOffset<__SIZEOF_POINTER__>().Int32Value()) // Offset of field Thread::tlsPtr_.thread_local_end. #define THREAD_LOCAL_END_OFFSET (THREAD_LOCAL_POS_OFFSET + __SIZEOF_POINTER__) ADD_TEST_EQ(THREAD_LOCAL_END_OFFSET, art::Thread::ThreadLocalEndOffset<__SIZEOF_POINTER__>().Int32Value()) -// Offset of field Thread::tlsPtr_.thread_local_objects. -#define THREAD_LOCAL_OBJECTS_OFFSET (THREAD_LOCAL_END_OFFSET + __SIZEOF_POINTER__) -ADD_TEST_EQ(THREAD_LOCAL_OBJECTS_OFFSET, - art::Thread::ThreadLocalObjectsOffset<__SIZEOF_POINTER__>().Int32Value()) // Offset of field Thread::tlsPtr_.mterp_current_ibase. -#define THREAD_CURRENT_IBASE_OFFSET (THREAD_LOCAL_OBJECTS_OFFSET + __SIZEOF_SIZE_T__) +#define THREAD_CURRENT_IBASE_OFFSET (THREAD_LOCAL_END_OFFSET + __SIZEOF_POINTER__) ADD_TEST_EQ(THREAD_CURRENT_IBASE_OFFSET, art::Thread::MterpCurrentIBaseOffset<__SIZEOF_POINTER__>().Int32Value()) // Offset of field Thread::tlsPtr_.mterp_default_ibase. diff --git a/runtime/entrypoints/quick/quick_entrypoints_list.h b/runtime/entrypoints/quick/quick_entrypoints_list.h index 30b639eaec..d6b7d9ef2e 100644 --- a/runtime/entrypoints/quick/quick_entrypoints_list.h +++ b/runtime/entrypoints/quick/quick_entrypoints_list.h @@ -164,7 +164,38 @@ V(NewStringFromStringBuilder, void) \ \ V(ReadBarrierJni, void, mirror::CompressedReference<mirror::Object>*, Thread*) \ - V(ReadBarrierMark, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg00, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg01, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg02, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg03, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg04, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg05, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg06, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg07, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg08, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg09, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg10, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg11, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg12, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg13, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg14, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg15, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg16, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg17, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg18, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg19, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg20, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg21, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg22, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg23, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg24, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg25, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg26, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg27, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg28, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg29, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg30, mirror::Object*, mirror::Object*) \ + V(ReadBarrierMarkReg31, mirror::Object*, mirror::Object*) \ V(ReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t) \ V(ReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*) diff --git a/runtime/entrypoints_order_test.cc b/runtime/entrypoints_order_test.cc index 7a624b211c..ffe41099ea 100644 --- a/runtime/entrypoints_order_test.cc +++ b/runtime/entrypoints_order_test.cc @@ -121,10 +121,10 @@ class EntrypointsOrderTest : public CommonRuntimeTest { // Skip across the entrypoints structures. - EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_start, thread_local_pos, sizeof(void*)); + EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_start, thread_local_objects, sizeof(void*)); + EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_objects, thread_local_pos, sizeof(size_t)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_pos, thread_local_end, sizeof(void*)); - EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_end, thread_local_objects, sizeof(void*)); - EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_objects, mterp_current_ibase, sizeof(void*)); + EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_end, mterp_current_ibase, sizeof(void*)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, mterp_current_ibase, mterp_default_ibase, sizeof(void*)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, mterp_default_ibase, mterp_alt_ibase, sizeof(void*)); EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, mterp_alt_ibase, rosalloc_runs, sizeof(void*)); @@ -324,8 +324,70 @@ class EntrypointsOrderTest : public CommonRuntimeTest { sizeof(void*)); EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pNewStringFromStringBuilder, pReadBarrierJni, sizeof(void*)); - EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierJni, pReadBarrierMark, sizeof(void*)); - EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMark, pReadBarrierSlow, sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierJni, pReadBarrierMarkReg00, sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg00, pReadBarrierMarkReg01, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg01, pReadBarrierMarkReg02, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg02, pReadBarrierMarkReg03, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg03, pReadBarrierMarkReg04, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg04, pReadBarrierMarkReg05, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg05, pReadBarrierMarkReg06, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg06, pReadBarrierMarkReg07, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg07, pReadBarrierMarkReg08, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg08, pReadBarrierMarkReg09, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg09, pReadBarrierMarkReg10, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg10, pReadBarrierMarkReg11, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg11, pReadBarrierMarkReg12, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg12, pReadBarrierMarkReg13, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg13, pReadBarrierMarkReg14, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg14, pReadBarrierMarkReg15, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg15, pReadBarrierMarkReg16, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg16, pReadBarrierMarkReg17, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg17, pReadBarrierMarkReg18, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg18, pReadBarrierMarkReg19, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg19, pReadBarrierMarkReg20, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg20, pReadBarrierMarkReg21, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg21, pReadBarrierMarkReg22, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg22, pReadBarrierMarkReg23, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg23, pReadBarrierMarkReg24, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg24, pReadBarrierMarkReg25, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg25, pReadBarrierMarkReg26, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg26, pReadBarrierMarkReg27, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg27, pReadBarrierMarkReg28, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg28, pReadBarrierMarkReg29, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg29, pReadBarrierMarkReg30, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg30, pReadBarrierMarkReg31, + sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierMarkReg31, pReadBarrierSlow, sizeof(void*)); EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierSlow, pReadBarrierForRootSlow, sizeof(void*)); diff --git a/runtime/oat.h b/runtime/oat.h index 6243660494..e506e3c476 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,7 +32,7 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - static constexpr uint8_t kOatVersion[] = { '0', '8', '2', '\0' }; + static constexpr uint8_t kOatVersion[] = { '0', '8', '3', '\0' }; static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; diff --git a/runtime/thread.cc b/runtime/thread.cc index b9ee4421b9..50f76da3df 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -2574,7 +2574,38 @@ void Thread::DumpThreadOffset(std::ostream& os, uint32_t offset) { QUICK_ENTRY_POINT_INFO(pNewStringFromStringBuffer) QUICK_ENTRY_POINT_INFO(pNewStringFromStringBuilder) QUICK_ENTRY_POINT_INFO(pReadBarrierJni) - QUICK_ENTRY_POINT_INFO(pReadBarrierMark) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg00) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg01) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg02) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg03) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg04) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg05) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg06) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg07) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg08) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg09) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg10) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg11) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg12) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg13) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg14) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg15) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg16) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg17) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg18) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg19) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg20) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg21) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg22) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg23) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg24) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg25) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg26) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg27) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg28) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg29) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg30) + QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg31) QUICK_ENTRY_POINT_INFO(pReadBarrierSlow) QUICK_ENTRY_POINT_INFO(pReadBarrierForRootSlow) #undef QUICK_ENTRY_POINT_INFO diff --git a/runtime/thread.h b/runtime/thread.h index ab2462535c..a3a4005347 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -1352,7 +1352,7 @@ class Thread { stacked_shadow_frame_record(nullptr), deoptimization_context_stack(nullptr), frame_id_to_shadow_frame(nullptr), name(nullptr), pthread_self(0), last_no_thread_suspension_cause(nullptr), thread_local_start(nullptr), - thread_local_pos(nullptr), thread_local_end(nullptr), thread_local_objects(0), + thread_local_objects(0), thread_local_pos(nullptr), thread_local_end(nullptr), mterp_current_ibase(nullptr), mterp_default_ibase(nullptr), mterp_alt_ibase(nullptr), thread_local_alloc_stack_top(nullptr), thread_local_alloc_stack_end(nullptr), nested_signal_state(nullptr), flip_function(nullptr), method_verifier(nullptr), @@ -1468,11 +1468,11 @@ class Thread { // Thread-local allocation pointer. uint8_t* thread_local_start; + size_t thread_local_objects; // thread_local_pos and thread_local_end must be consecutive for ldrd and are 8 byte aligned for // potentially better performance. uint8_t* thread_local_pos; uint8_t* thread_local_end; - size_t thread_local_objects; // Mterp jump table bases. void* mterp_current_ibase; |