diff options
Diffstat (limited to 'compiler/optimizing')
20 files changed, 2584 insertions, 1381 deletions
diff --git a/compiler/optimizing/builder.h b/compiler/optimizing/builder.h index ca71c32802..c3979f3dd1 100644 --- a/compiler/optimizing/builder.h +++ b/compiler/optimizing/builder.h @@ -90,9 +90,8 @@ class HGraphBuilder : public ValueObject { static constexpr const char* kBuilderPassName = "builder"; - // The number of entries in a packed switch before we use a jump table or specified - // compare/jump series. - static constexpr uint16_t kSmallSwitchThreshold = 3; + // The number of entries in a packed switch before we use a jump table. + static constexpr uint16_t kSmallSwitchThreshold = 5; private: // Analyzes the dex instruction and adds HInstruction to the graph diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index a941935446..3630dbec24 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -59,7 +59,7 @@ static constexpr SRegister kFpuCalleeSaves[] = // S registers. Therefore there is no need to block it. static constexpr DRegister DTMP = D31; -static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; +static constexpr uint32_t kPackedSwitchJumpTableThreshold = 6; #define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmWordSize, x).Int32Value() @@ -3363,15 +3363,16 @@ void InstructionCodeGeneratorARM::HandleRotate(HRor* ror) { } default: LOG(FATAL) << "Unexpected operation type " << type; + UNREACHABLE(); } } void LocationsBuilderARM::VisitRor(HRor* op) { - HandleRotate(op); + HandleRotate(op); } void InstructionCodeGeneratorARM::VisitRor(HRor* op) { - HandleRotate(op); + HandleRotate(op); } void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { @@ -6249,7 +6250,7 @@ void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); - if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && + if (switch_instr->GetNumEntries() >= kPackedSwitchJumpTableThreshold && codegen_->GetAssembler()->IsThumb()) { locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. if (switch_instr->GetStartValue() != 0) { @@ -6265,30 +6266,12 @@ void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) Register value_reg = locations->InAt(0).AsRegister<Register>(); HBasicBlock* default_block = switch_instr->GetDefaultBlock(); - if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { + if (num_entries < kPackedSwitchJumpTableThreshold || !codegen_->GetAssembler()->IsThumb()) { // Create a series of compare/jumps. - Register temp_reg = IP; - // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store - // the immediate, because IP is used as the destination register. For the other - // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, - // and they can be encoded in the instruction without making use of IP register. - __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - // Jump to successors[0] if value == lower_bound. - __ b(codegen_->GetLabelOf(successors[0]), EQ); - int32_t last_index = 0; - for (; num_entries - last_index > 2; last_index += 2) { - __ AddConstantSetFlags(temp_reg, temp_reg, -2); - // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. - __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); - // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. - __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); - } - if (num_entries - last_index == 2) { - // The last missing case_value. - GenerateCompareWithImmediate(temp_reg, 1); - __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); + for (uint32_t i = 0; i < num_entries; i++) { + GenerateCompareWithImmediate(value_reg, lower_bound + i); + __ b(codegen_->GetLabelOf(successors[i]), EQ); } // And the default for any other value. diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index b7efbe351c..451470f271 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -71,10 +71,10 @@ using helpers::ARM64EncodableConstantOrRegister; using helpers::ArtVixlRegCodeCoherentForRegSet; static constexpr int kCurrentMethodStackOffset = 0; -// The compare/jump sequence will generate about (1.5 * num_entries + 3) instructions. While jump +// The compare/jump sequence will generate about (2 * num_entries + 1) instructions. While jump // table version generates 7 instructions and num_entries literals. Compare/jump sequence will // generates less code/data with a small num_entries. -static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; +static constexpr uint32_t kPackedSwitchJumpTableThreshold = 6; inline Condition ARM64Condition(IfCondition cond) { switch (cond) { @@ -546,7 +546,7 @@ class ArraySetSlowPathARM64 : public SlowPathCodeARM64 { void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { uint32_t num_entries = switch_instr_->GetNumEntries(); - DCHECK_GE(num_entries, kPackedSwitchCompareJumpThreshold); + DCHECK_GE(num_entries, kPackedSwitchJumpTableThreshold); // We are about to use the assembler to place literals directly. Make sure we have enough // underlying code buffer and we have generated the jump table with right size. @@ -4243,6 +4243,7 @@ void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) { default: LOG(FATAL) << "Unexpected rem type " << type; + UNREACHABLE(); } } @@ -4317,6 +4318,7 @@ void LocationsBuilderARM64::VisitStoreLocal(HStoreLocal* store) { default: LOG(FATAL) << "Unimplemented local type " << field_type; + UNREACHABLE(); } } @@ -4580,29 +4582,20 @@ void InstructionCodeGeneratorARM64::VisitPackedSwitch(HPackedSwitch* switch_inst // ranges and emit the tables only as required. static constexpr int32_t kJumpTableInstructionThreshold = 1* MB / kMaxExpectedSizePerHInstruction; - if (num_entries <= kPackedSwitchCompareJumpThreshold || + if (num_entries < kPackedSwitchJumpTableThreshold || // Current instruction id is an upper bound of the number of HIRs in the graph. GetGraph()->GetCurrentInstructionId() > kJumpTableInstructionThreshold) { // Create a series of compare/jumps. - UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); - Register temp = temps.AcquireW(); - __ Subs(temp, value_reg, Operand(lower_bound)); - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - // Jump to successors[0] if value == lower_bound. - __ B(eq, codegen_->GetLabelOf(successors[0])); - int32_t last_index = 0; - for (; num_entries - last_index > 2; last_index += 2) { - __ Subs(temp, temp, Operand(2)); - // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. - __ B(lo, codegen_->GetLabelOf(successors[last_index + 1])); - // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. - __ B(eq, codegen_->GetLabelOf(successors[last_index + 2])); - } - if (num_entries - last_index == 2) { - // The last missing case_value. - __ Cmp(temp, Operand(1)); - __ B(eq, codegen_->GetLabelOf(successors[last_index + 1])); + for (uint32_t i = 0; i < num_entries; i++) { + int32_t case_value = lower_bound + i; + vixl::Label* succ = codegen_->GetLabelOf(successors[i]); + if (case_value == 0) { + __ Cbz(value_reg, succ); + } else { + __ Cmp(value_reg, Operand(case_value)); + __ B(eq, succ); + } } // And the default for any other value. diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index ce7cbcd9d6..5dc101b199 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -1956,11 +1956,8 @@ void InstructionCodeGeneratorMIPS::VisitClinitCheck(HClinitCheck* check) { void LocationsBuilderMIPS::VisitCompare(HCompare* compare) { Primitive::Type in_type = compare->InputAt(0)->GetType(); - LocationSummary::CallKind call_kind = Primitive::IsFloatingPointType(in_type) - ? LocationSummary::kCall - : LocationSummary::kNoCall; - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(compare, call_kind); + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); switch (in_type) { case Primitive::kPrimLong: @@ -1971,13 +1968,11 @@ void LocationsBuilderMIPS::VisitCompare(HCompare* compare) { break; case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); - locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); - locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt)); + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); break; - } default: LOG(FATAL) << "Unexpected type for compare operation " << in_type; @@ -1986,7 +1981,10 @@ void LocationsBuilderMIPS::VisitCompare(HCompare* compare) { void InstructionCodeGeneratorMIPS::VisitCompare(HCompare* instruction) { LocationSummary* locations = instruction->GetLocations(); + Register res = locations->Out().AsRegister<Register>(); Primitive::Type in_type = instruction->InputAt(0)->GetType(); + bool gt_bias = instruction->IsGtBias(); + bool isR6 = codegen_->GetInstructionSetFeatures().IsR6(); // 0 if: left == right // 1 if: left > right @@ -1994,7 +1992,6 @@ void InstructionCodeGeneratorMIPS::VisitCompare(HCompare* instruction) { switch (in_type) { case Primitive::kPrimLong: { MipsLabel done; - Register res = locations->Out().AsRegister<Register>(); Register lhs_high = locations->InAt(0).AsRegisterPairHigh<Register>(); Register lhs_low = locations->InAt(0).AsRegisterPairLow<Register>(); Register rhs_high = locations->InAt(1).AsRegisterPairHigh<Register>(); @@ -2011,45 +2008,82 @@ void InstructionCodeGeneratorMIPS::VisitCompare(HCompare* instruction) { break; } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - int32_t entry_point_offset; - bool direct; - if (in_type == Primitive::kPrimFloat) { - if (instruction->IsGtBias()) { - entry_point_offset = QUICK_ENTRY_POINT(pCmpgFloat); - direct = IsDirectEntrypoint(kQuickCmpgFloat); + case Primitive::kPrimFloat: { + FRegister lhs = locations->InAt(0).AsFpuRegister<FRegister>(); + FRegister rhs = locations->InAt(1).AsFpuRegister<FRegister>(); + MipsLabel done; + if (isR6) { + __ CmpEqS(FTMP, lhs, rhs); + __ LoadConst32(res, 0); + __ Bc1nez(FTMP, &done); + if (gt_bias) { + __ CmpLtS(FTMP, lhs, rhs); + __ LoadConst32(res, -1); + __ Bc1nez(FTMP, &done); + __ LoadConst32(res, 1); } else { - entry_point_offset = QUICK_ENTRY_POINT(pCmplFloat); - direct = IsDirectEntrypoint(kQuickCmplFloat); + __ CmpLtS(FTMP, rhs, lhs); + __ LoadConst32(res, 1); + __ Bc1nez(FTMP, &done); + __ LoadConst32(res, -1); } } else { - if (instruction->IsGtBias()) { - entry_point_offset = QUICK_ENTRY_POINT(pCmpgDouble); - direct = IsDirectEntrypoint(kQuickCmpgDouble); + if (gt_bias) { + __ ColtS(0, lhs, rhs); + __ LoadConst32(res, -1); + __ Bc1t(0, &done); + __ CeqS(0, lhs, rhs); + __ LoadConst32(res, 1); + __ Movt(res, ZERO, 0); } else { - entry_point_offset = QUICK_ENTRY_POINT(pCmplDouble); - direct = IsDirectEntrypoint(kQuickCmplDouble); + __ ColtS(0, rhs, lhs); + __ LoadConst32(res, 1); + __ Bc1t(0, &done); + __ CeqS(0, lhs, rhs); + __ LoadConst32(res, -1); + __ Movt(res, ZERO, 0); } } - codegen_->InvokeRuntime(entry_point_offset, - instruction, - instruction->GetDexPc(), - nullptr, - direct); - if (in_type == Primitive::kPrimFloat) { - if (instruction->IsGtBias()) { - CheckEntrypointTypes<kQuickCmpgFloat, int32_t, float, float>(); + __ Bind(&done); + break; + } + case Primitive::kPrimDouble: { + FRegister lhs = locations->InAt(0).AsFpuRegister<FRegister>(); + FRegister rhs = locations->InAt(1).AsFpuRegister<FRegister>(); + MipsLabel done; + if (isR6) { + __ CmpEqD(FTMP, lhs, rhs); + __ LoadConst32(res, 0); + __ Bc1nez(FTMP, &done); + if (gt_bias) { + __ CmpLtD(FTMP, lhs, rhs); + __ LoadConst32(res, -1); + __ Bc1nez(FTMP, &done); + __ LoadConst32(res, 1); } else { - CheckEntrypointTypes<kQuickCmplFloat, int32_t, float, float>(); + __ CmpLtD(FTMP, rhs, lhs); + __ LoadConst32(res, 1); + __ Bc1nez(FTMP, &done); + __ LoadConst32(res, -1); } } else { - if (instruction->IsGtBias()) { - CheckEntrypointTypes<kQuickCmpgDouble, int32_t, double, double>(); + if (gt_bias) { + __ ColtD(0, lhs, rhs); + __ LoadConst32(res, -1); + __ Bc1t(0, &done); + __ CeqD(0, lhs, rhs); + __ LoadConst32(res, 1); + __ Movt(res, ZERO, 0); } else { - CheckEntrypointTypes<kQuickCmplDouble, int32_t, double, double>(); + __ ColtD(0, rhs, lhs); + __ LoadConst32(res, 1); + __ Bc1t(0, &done); + __ CeqD(0, lhs, rhs); + __ LoadConst32(res, -1); + __ Movt(res, ZERO, 0); } } + __ Bind(&done); break; } @@ -2060,8 +2094,19 @@ void InstructionCodeGeneratorMIPS::VisitCompare(HCompare* instruction) { void LocationsBuilderMIPS::VisitCondition(HCondition* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + switch (instruction->InputAt(0)->GetType()) { + default: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + break; + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + break; + } if (instruction->NeedsMaterialization()) { locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } @@ -2071,151 +2116,45 @@ void InstructionCodeGeneratorMIPS::VisitCondition(HCondition* instruction) { if (!instruction->NeedsMaterialization()) { return; } - // TODO: generalize to long - DCHECK_NE(instruction->InputAt(0)->GetType(), Primitive::kPrimLong); + Primitive::Type type = instruction->InputAt(0)->GetType(); LocationSummary* locations = instruction->GetLocations(); Register dst = locations->Out().AsRegister<Register>(); + MipsLabel true_label; - Register lhs = locations->InAt(0).AsRegister<Register>(); - Location rhs_location = locations->InAt(1); - - Register rhs_reg = ZERO; - int64_t rhs_imm = 0; - bool use_imm = rhs_location.IsConstant(); - if (use_imm) { - rhs_imm = CodeGenerator::GetInt32ValueOf(rhs_location.GetConstant()); - } else { - rhs_reg = rhs_location.AsRegister<Register>(); - } - - IfCondition if_cond = instruction->GetCondition(); + switch (type) { + default: + // Integer case. + GenerateIntCompare(instruction->GetCondition(), locations); + return; - switch (if_cond) { - case kCondEQ: - case kCondNE: - if (use_imm && IsUint<16>(rhs_imm)) { - __ Xori(dst, lhs, rhs_imm); - } else { - if (use_imm) { - rhs_reg = TMP; - __ LoadConst32(rhs_reg, rhs_imm); - } - __ Xor(dst, lhs, rhs_reg); - } - if (if_cond == kCondEQ) { - __ Sltiu(dst, dst, 1); - } else { - __ Sltu(dst, ZERO, dst); - } + case Primitive::kPrimLong: + // TODO: don't use branches. + GenerateLongCompareAndBranch(instruction->GetCondition(), locations, &true_label); break; - case kCondLT: - case kCondGE: - if (use_imm && IsInt<16>(rhs_imm)) { - __ Slti(dst, lhs, rhs_imm); - } else { - if (use_imm) { - rhs_reg = TMP; - __ LoadConst32(rhs_reg, rhs_imm); - } - __ Slt(dst, lhs, rhs_reg); - } - if (if_cond == kCondGE) { - // Simulate lhs >= rhs via !(lhs < rhs) since there's - // only the slt instruction but no sge. - __ Xori(dst, dst, 1); - } + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + // TODO: don't use branches. + GenerateFpCompareAndBranch(instruction->GetCondition(), + instruction->IsGtBias(), + type, + locations, + &true_label); break; + } - case kCondLE: - case kCondGT: - if (use_imm && IsInt<16>(rhs_imm + 1)) { - // Simulate lhs <= rhs via lhs < rhs + 1. - __ Slti(dst, lhs, rhs_imm + 1); - if (if_cond == kCondGT) { - // Simulate lhs > rhs via !(lhs <= rhs) since there's - // only the slti instruction but no sgti. - __ Xori(dst, dst, 1); - } - } else { - if (use_imm) { - rhs_reg = TMP; - __ LoadConst32(rhs_reg, rhs_imm); - } - __ Slt(dst, rhs_reg, lhs); - if (if_cond == kCondLE) { - // Simulate lhs <= rhs via !(rhs < lhs) since there's - // only the slt instruction but no sle. - __ Xori(dst, dst, 1); - } - } - break; + // Convert the branches into the result. + MipsLabel done; - case kCondB: - case kCondAE: - // Use sltiu instruction if rhs_imm is in range [0, 32767] or in - // [max_unsigned - 32767 = 0xffff8000, max_unsigned = 0xffffffff]. - if (use_imm && - (IsUint<15>(rhs_imm) || - IsUint<15>(rhs_imm - (MaxInt<uint64_t>(32) - MaxInt<uint64_t>(15))))) { - if (IsUint<15>(rhs_imm)) { - __ Sltiu(dst, lhs, rhs_imm); - } else { - // 16-bit value (in range [0x8000, 0xffff]) passed to sltiu is sign-extended - // and then used as unsigned integer (range [0xffff8000, 0xffffffff]). - __ Sltiu(dst, lhs, rhs_imm - (MaxInt<uint64_t>(32) - MaxInt<uint64_t>(16))); - } - } else { - if (use_imm) { - rhs_reg = TMP; - __ LoadConst32(rhs_reg, rhs_imm); - } - __ Sltu(dst, lhs, rhs_reg); - } - if (if_cond == kCondAE) { - // Simulate lhs >= rhs via !(lhs < rhs) since there's - // only the sltu instruction but no sgeu. - __ Xori(dst, dst, 1); - } - break; + // False case: result = 0. + __ LoadConst32(dst, 0); + __ B(&done); - case kCondBE: - case kCondA: - // Use sltiu instruction if rhs_imm is in range [0, 32766] or in - // [max_unsigned - 32767 - 1 = 0xffff7fff, max_unsigned - 1 = 0xfffffffe]. - // lhs <= rhs is simulated via lhs < rhs + 1. - if (use_imm && (rhs_imm != -1) && - (IsUint<15>(rhs_imm + 1) || - IsUint<15>(rhs_imm + 1 - (MaxInt<uint64_t>(32) - MaxInt<uint64_t>(15))))) { - if (IsUint<15>(rhs_imm + 1)) { - // Simulate lhs <= rhs via lhs < rhs + 1. - __ Sltiu(dst, lhs, rhs_imm + 1); - } else { - // 16-bit value (in range [0x8000, 0xffff]) passed to sltiu is sign-extended - // and then used as unsigned integer (range [0xffff8000, 0xffffffff] where rhs_imm - // is in range [0xffff7fff, 0xfffffffe] since lhs <= rhs is simulated via lhs < rhs + 1). - __ Sltiu(dst, lhs, rhs_imm + 1 - (MaxInt<uint64_t>(32) - MaxInt<uint64_t>(16))); - } - if (if_cond == kCondA) { - // Simulate lhs > rhs via !(lhs <= rhs) since there's - // only the sltiu instruction but no sgtiu. - __ Xori(dst, dst, 1); - } - } else { - if (use_imm) { - rhs_reg = TMP; - __ LoadConst32(rhs_reg, rhs_imm); - } - __ Sltu(dst, rhs_reg, lhs); - if (if_cond == kCondBE) { - // Simulate lhs <= rhs via !(rhs < lhs) since there's - // only the sltu instruction but no sleu. - __ Xori(dst, dst, 1); - } - } - break; - } + // True case: result = 1. + __ Bind(&true_label); + __ LoadConst32(dst, 1); + __ Bind(&done); } void InstructionCodeGeneratorMIPS::DivRemOneOrMinusOne(HBinaryOperation* instruction) { @@ -2574,6 +2513,627 @@ void InstructionCodeGeneratorMIPS::VisitTryBoundary(HTryBoundary* try_boundary) } } +void InstructionCodeGeneratorMIPS::GenerateIntCompare(IfCondition cond, + LocationSummary* locations) { + Register dst = locations->Out().AsRegister<Register>(); + Register lhs = locations->InAt(0).AsRegister<Register>(); + Location rhs_location = locations->InAt(1); + Register rhs_reg = ZERO; + int64_t rhs_imm = 0; + bool use_imm = rhs_location.IsConstant(); + if (use_imm) { + rhs_imm = CodeGenerator::GetInt32ValueOf(rhs_location.GetConstant()); + } else { + rhs_reg = rhs_location.AsRegister<Register>(); + } + + switch (cond) { + case kCondEQ: + case kCondNE: + if (use_imm && IsUint<16>(rhs_imm)) { + __ Xori(dst, lhs, rhs_imm); + } else { + if (use_imm) { + rhs_reg = TMP; + __ LoadConst32(rhs_reg, rhs_imm); + } + __ Xor(dst, lhs, rhs_reg); + } + if (cond == kCondEQ) { + __ Sltiu(dst, dst, 1); + } else { + __ Sltu(dst, ZERO, dst); + } + break; + + case kCondLT: + case kCondGE: + if (use_imm && IsInt<16>(rhs_imm)) { + __ Slti(dst, lhs, rhs_imm); + } else { + if (use_imm) { + rhs_reg = TMP; + __ LoadConst32(rhs_reg, rhs_imm); + } + __ Slt(dst, lhs, rhs_reg); + } + if (cond == kCondGE) { + // Simulate lhs >= rhs via !(lhs < rhs) since there's + // only the slt instruction but no sge. + __ Xori(dst, dst, 1); + } + break; + + case kCondLE: + case kCondGT: + if (use_imm && IsInt<16>(rhs_imm + 1)) { + // Simulate lhs <= rhs via lhs < rhs + 1. + __ Slti(dst, lhs, rhs_imm + 1); + if (cond == kCondGT) { + // Simulate lhs > rhs via !(lhs <= rhs) since there's + // only the slti instruction but no sgti. + __ Xori(dst, dst, 1); + } + } else { + if (use_imm) { + rhs_reg = TMP; + __ LoadConst32(rhs_reg, rhs_imm); + } + __ Slt(dst, rhs_reg, lhs); + if (cond == kCondLE) { + // Simulate lhs <= rhs via !(rhs < lhs) since there's + // only the slt instruction but no sle. + __ Xori(dst, dst, 1); + } + } + break; + + case kCondB: + case kCondAE: + if (use_imm && IsInt<16>(rhs_imm)) { + // Sltiu sign-extends its 16-bit immediate operand before + // the comparison and thus lets us compare directly with + // unsigned values in the ranges [0, 0x7fff] and + // [0xffff8000, 0xffffffff]. + __ Sltiu(dst, lhs, rhs_imm); + } else { + if (use_imm) { + rhs_reg = TMP; + __ LoadConst32(rhs_reg, rhs_imm); + } + __ Sltu(dst, lhs, rhs_reg); + } + if (cond == kCondAE) { + // Simulate lhs >= rhs via !(lhs < rhs) since there's + // only the sltu instruction but no sgeu. + __ Xori(dst, dst, 1); + } + break; + + case kCondBE: + case kCondA: + if (use_imm && (rhs_imm != -1) && IsInt<16>(rhs_imm + 1)) { + // Simulate lhs <= rhs via lhs < rhs + 1. + // Note that this only works if rhs + 1 does not overflow + // to 0, hence the check above. + // Sltiu sign-extends its 16-bit immediate operand before + // the comparison and thus lets us compare directly with + // unsigned values in the ranges [0, 0x7fff] and + // [0xffff8000, 0xffffffff]. + __ Sltiu(dst, lhs, rhs_imm + 1); + if (cond == kCondA) { + // Simulate lhs > rhs via !(lhs <= rhs) since there's + // only the sltiu instruction but no sgtiu. + __ Xori(dst, dst, 1); + } + } else { + if (use_imm) { + rhs_reg = TMP; + __ LoadConst32(rhs_reg, rhs_imm); + } + __ Sltu(dst, rhs_reg, lhs); + if (cond == kCondBE) { + // Simulate lhs <= rhs via !(rhs < lhs) since there's + // only the sltu instruction but no sleu. + __ Xori(dst, dst, 1); + } + } + break; + } +} + +void InstructionCodeGeneratorMIPS::GenerateIntCompareAndBranch(IfCondition cond, + LocationSummary* locations, + MipsLabel* label) { + Register lhs = locations->InAt(0).AsRegister<Register>(); + Location rhs_location = locations->InAt(1); + Register rhs_reg = ZERO; + int32_t rhs_imm = 0; + bool use_imm = rhs_location.IsConstant(); + if (use_imm) { + rhs_imm = CodeGenerator::GetInt32ValueOf(rhs_location.GetConstant()); + } else { + rhs_reg = rhs_location.AsRegister<Register>(); + } + + if (use_imm && rhs_imm == 0) { + switch (cond) { + case kCondEQ: + case kCondBE: // <= 0 if zero + __ Beqz(lhs, label); + break; + case kCondNE: + case kCondA: // > 0 if non-zero + __ Bnez(lhs, label); + break; + case kCondLT: + __ Bltz(lhs, label); + break; + case kCondGE: + __ Bgez(lhs, label); + break; + case kCondLE: + __ Blez(lhs, label); + break; + case kCondGT: + __ Bgtz(lhs, label); + break; + case kCondB: // always false + break; + case kCondAE: // always true + __ B(label); + break; + } + } else { + if (use_imm) { + // TODO: more efficient comparison with 16-bit constants without loading them into TMP. + rhs_reg = TMP; + __ LoadConst32(rhs_reg, rhs_imm); + } + switch (cond) { + case kCondEQ: + __ Beq(lhs, rhs_reg, label); + break; + case kCondNE: + __ Bne(lhs, rhs_reg, label); + break; + case kCondLT: + __ Blt(lhs, rhs_reg, label); + break; + case kCondGE: + __ Bge(lhs, rhs_reg, label); + break; + case kCondLE: + __ Bge(rhs_reg, lhs, label); + break; + case kCondGT: + __ Blt(rhs_reg, lhs, label); + break; + case kCondB: + __ Bltu(lhs, rhs_reg, label); + break; + case kCondAE: + __ Bgeu(lhs, rhs_reg, label); + break; + case kCondBE: + __ Bgeu(rhs_reg, lhs, label); + break; + case kCondA: + __ Bltu(rhs_reg, lhs, label); + break; + } + } +} + +void InstructionCodeGeneratorMIPS::GenerateLongCompareAndBranch(IfCondition cond, + LocationSummary* locations, + MipsLabel* label) { + Register lhs_high = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register lhs_low = locations->InAt(0).AsRegisterPairLow<Register>(); + Location rhs_location = locations->InAt(1); + Register rhs_high = ZERO; + Register rhs_low = ZERO; + int64_t imm = 0; + uint32_t imm_high = 0; + uint32_t imm_low = 0; + bool use_imm = rhs_location.IsConstant(); + if (use_imm) { + imm = rhs_location.GetConstant()->AsLongConstant()->GetValue(); + imm_high = High32Bits(imm); + imm_low = Low32Bits(imm); + } else { + rhs_high = rhs_location.AsRegisterPairHigh<Register>(); + rhs_low = rhs_location.AsRegisterPairLow<Register>(); + } + + if (use_imm && imm == 0) { + switch (cond) { + case kCondEQ: + case kCondBE: // <= 0 if zero + __ Or(TMP, lhs_high, lhs_low); + __ Beqz(TMP, label); + break; + case kCondNE: + case kCondA: // > 0 if non-zero + __ Or(TMP, lhs_high, lhs_low); + __ Bnez(TMP, label); + break; + case kCondLT: + __ Bltz(lhs_high, label); + break; + case kCondGE: + __ Bgez(lhs_high, label); + break; + case kCondLE: + __ Or(TMP, lhs_high, lhs_low); + __ Sra(AT, lhs_high, 31); + __ Bgeu(AT, TMP, label); + break; + case kCondGT: + __ Or(TMP, lhs_high, lhs_low); + __ Sra(AT, lhs_high, 31); + __ Bltu(AT, TMP, label); + break; + case kCondB: // always false + break; + case kCondAE: // always true + __ B(label); + break; + } + } else if (use_imm) { + // TODO: more efficient comparison with constants without loading them into TMP/AT. + switch (cond) { + case kCondEQ: + __ LoadConst32(TMP, imm_high); + __ Xor(TMP, TMP, lhs_high); + __ LoadConst32(AT, imm_low); + __ Xor(AT, AT, lhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondNE: + __ LoadConst32(TMP, imm_high); + __ Xor(TMP, TMP, lhs_high); + __ LoadConst32(AT, imm_low); + __ Xor(AT, AT, lhs_low); + __ Or(TMP, TMP, AT); + __ Bnez(TMP, label); + break; + case kCondLT: + __ LoadConst32(TMP, imm_high); + __ Blt(lhs_high, TMP, label); + __ Slt(TMP, TMP, lhs_high); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, lhs_low, AT); + __ Blt(TMP, AT, label); + break; + case kCondGE: + __ LoadConst32(TMP, imm_high); + __ Blt(TMP, lhs_high, label); + __ Slt(TMP, lhs_high, TMP); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, lhs_low, AT); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondLE: + __ LoadConst32(TMP, imm_high); + __ Blt(lhs_high, TMP, label); + __ Slt(TMP, TMP, lhs_high); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, AT, lhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondGT: + __ LoadConst32(TMP, imm_high); + __ Blt(TMP, lhs_high, label); + __ Slt(TMP, lhs_high, TMP); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, AT, lhs_low); + __ Blt(TMP, AT, label); + break; + case kCondB: + __ LoadConst32(TMP, imm_high); + __ Bltu(lhs_high, TMP, label); + __ Sltu(TMP, TMP, lhs_high); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, lhs_low, AT); + __ Blt(TMP, AT, label); + break; + case kCondAE: + __ LoadConst32(TMP, imm_high); + __ Bltu(TMP, lhs_high, label); + __ Sltu(TMP, lhs_high, TMP); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, lhs_low, AT); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondBE: + __ LoadConst32(TMP, imm_high); + __ Bltu(lhs_high, TMP, label); + __ Sltu(TMP, TMP, lhs_high); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, AT, lhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondA: + __ LoadConst32(TMP, imm_high); + __ Bltu(TMP, lhs_high, label); + __ Sltu(TMP, lhs_high, TMP); + __ LoadConst32(AT, imm_low); + __ Sltu(AT, AT, lhs_low); + __ Blt(TMP, AT, label); + break; + } + } else { + switch (cond) { + case kCondEQ: + __ Xor(TMP, lhs_high, rhs_high); + __ Xor(AT, lhs_low, rhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondNE: + __ Xor(TMP, lhs_high, rhs_high); + __ Xor(AT, lhs_low, rhs_low); + __ Or(TMP, TMP, AT); + __ Bnez(TMP, label); + break; + case kCondLT: + __ Blt(lhs_high, rhs_high, label); + __ Slt(TMP, rhs_high, lhs_high); + __ Sltu(AT, lhs_low, rhs_low); + __ Blt(TMP, AT, label); + break; + case kCondGE: + __ Blt(rhs_high, lhs_high, label); + __ Slt(TMP, lhs_high, rhs_high); + __ Sltu(AT, lhs_low, rhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondLE: + __ Blt(lhs_high, rhs_high, label); + __ Slt(TMP, rhs_high, lhs_high); + __ Sltu(AT, rhs_low, lhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondGT: + __ Blt(rhs_high, lhs_high, label); + __ Slt(TMP, lhs_high, rhs_high); + __ Sltu(AT, rhs_low, lhs_low); + __ Blt(TMP, AT, label); + break; + case kCondB: + __ Bltu(lhs_high, rhs_high, label); + __ Sltu(TMP, rhs_high, lhs_high); + __ Sltu(AT, lhs_low, rhs_low); + __ Blt(TMP, AT, label); + break; + case kCondAE: + __ Bltu(rhs_high, lhs_high, label); + __ Sltu(TMP, lhs_high, rhs_high); + __ Sltu(AT, lhs_low, rhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondBE: + __ Bltu(lhs_high, rhs_high, label); + __ Sltu(TMP, rhs_high, lhs_high); + __ Sltu(AT, rhs_low, lhs_low); + __ Or(TMP, TMP, AT); + __ Beqz(TMP, label); + break; + case kCondA: + __ Bltu(rhs_high, lhs_high, label); + __ Sltu(TMP, lhs_high, rhs_high); + __ Sltu(AT, rhs_low, lhs_low); + __ Blt(TMP, AT, label); + break; + } + } +} + +void InstructionCodeGeneratorMIPS::GenerateFpCompareAndBranch(IfCondition cond, + bool gt_bias, + Primitive::Type type, + LocationSummary* locations, + MipsLabel* label) { + FRegister lhs = locations->InAt(0).AsFpuRegister<FRegister>(); + FRegister rhs = locations->InAt(1).AsFpuRegister<FRegister>(); + bool isR6 = codegen_->GetInstructionSetFeatures().IsR6(); + if (type == Primitive::kPrimFloat) { + if (isR6) { + switch (cond) { + case kCondEQ: + __ CmpEqS(FTMP, lhs, rhs); + __ Bc1nez(FTMP, label); + break; + case kCondNE: + __ CmpEqS(FTMP, lhs, rhs); + __ Bc1eqz(FTMP, label); + break; + case kCondLT: + if (gt_bias) { + __ CmpLtS(FTMP, lhs, rhs); + } else { + __ CmpUltS(FTMP, lhs, rhs); + } + __ Bc1nez(FTMP, label); + break; + case kCondLE: + if (gt_bias) { + __ CmpLeS(FTMP, lhs, rhs); + } else { + __ CmpUleS(FTMP, lhs, rhs); + } + __ Bc1nez(FTMP, label); + break; + case kCondGT: + if (gt_bias) { + __ CmpUltS(FTMP, rhs, lhs); + } else { + __ CmpLtS(FTMP, rhs, lhs); + } + __ Bc1nez(FTMP, label); + break; + case kCondGE: + if (gt_bias) { + __ CmpUleS(FTMP, rhs, lhs); + } else { + __ CmpLeS(FTMP, rhs, lhs); + } + __ Bc1nez(FTMP, label); + break; + default: + LOG(FATAL) << "Unexpected non-floating-point condition"; + } + } else { + switch (cond) { + case kCondEQ: + __ CeqS(0, lhs, rhs); + __ Bc1t(0, label); + break; + case kCondNE: + __ CeqS(0, lhs, rhs); + __ Bc1f(0, label); + break; + case kCondLT: + if (gt_bias) { + __ ColtS(0, lhs, rhs); + } else { + __ CultS(0, lhs, rhs); + } + __ Bc1t(0, label); + break; + case kCondLE: + if (gt_bias) { + __ ColeS(0, lhs, rhs); + } else { + __ CuleS(0, lhs, rhs); + } + __ Bc1t(0, label); + break; + case kCondGT: + if (gt_bias) { + __ CultS(0, rhs, lhs); + } else { + __ ColtS(0, rhs, lhs); + } + __ Bc1t(0, label); + break; + case kCondGE: + if (gt_bias) { + __ CuleS(0, rhs, lhs); + } else { + __ ColeS(0, rhs, lhs); + } + __ Bc1t(0, label); + break; + default: + LOG(FATAL) << "Unexpected non-floating-point condition"; + } + } + } else { + DCHECK_EQ(type, Primitive::kPrimDouble); + if (isR6) { + switch (cond) { + case kCondEQ: + __ CmpEqD(FTMP, lhs, rhs); + __ Bc1nez(FTMP, label); + break; + case kCondNE: + __ CmpEqD(FTMP, lhs, rhs); + __ Bc1eqz(FTMP, label); + break; + case kCondLT: + if (gt_bias) { + __ CmpLtD(FTMP, lhs, rhs); + } else { + __ CmpUltD(FTMP, lhs, rhs); + } + __ Bc1nez(FTMP, label); + break; + case kCondLE: + if (gt_bias) { + __ CmpLeD(FTMP, lhs, rhs); + } else { + __ CmpUleD(FTMP, lhs, rhs); + } + __ Bc1nez(FTMP, label); + break; + case kCondGT: + if (gt_bias) { + __ CmpUltD(FTMP, rhs, lhs); + } else { + __ CmpLtD(FTMP, rhs, lhs); + } + __ Bc1nez(FTMP, label); + break; + case kCondGE: + if (gt_bias) { + __ CmpUleD(FTMP, rhs, lhs); + } else { + __ CmpLeD(FTMP, rhs, lhs); + } + __ Bc1nez(FTMP, label); + break; + default: + LOG(FATAL) << "Unexpected non-floating-point condition"; + } + } else { + switch (cond) { + case kCondEQ: + __ CeqD(0, lhs, rhs); + __ Bc1t(0, label); + break; + case kCondNE: + __ CeqD(0, lhs, rhs); + __ Bc1f(0, label); + break; + case kCondLT: + if (gt_bias) { + __ ColtD(0, lhs, rhs); + } else { + __ CultD(0, lhs, rhs); + } + __ Bc1t(0, label); + break; + case kCondLE: + if (gt_bias) { + __ ColeD(0, lhs, rhs); + } else { + __ CuleD(0, lhs, rhs); + } + __ Bc1t(0, label); + break; + case kCondGT: + if (gt_bias) { + __ CultD(0, rhs, lhs); + } else { + __ ColtD(0, rhs, lhs); + } + __ Bc1t(0, label); + break; + case kCondGE: + if (gt_bias) { + __ CuleD(0, rhs, lhs); + } else { + __ ColeD(0, rhs, lhs); + } + __ Bc1t(0, label); + break; + default: + LOG(FATAL) << "Unexpected non-floating-point condition"; + } + } + } +} + void InstructionCodeGeneratorMIPS::GenerateTestAndBranch(HInstruction* instruction, size_t condition_input_index, MipsLabel* true_target, @@ -2610,7 +3170,7 @@ void InstructionCodeGeneratorMIPS::GenerateTestAndBranch(HInstruction* instructi // The condition instruction has been materialized, compare the output to 0. Location cond_val = instruction->GetLocations()->InAt(condition_input_index); DCHECK(cond_val.IsRegister()); - if (true_target == nullptr) { + if (true_target == nullptr) { __ Beqz(cond_val.AsRegister<Register>(), false_target); } else { __ Bnez(cond_val.AsRegister<Register>(), true_target); @@ -2619,98 +3179,27 @@ void InstructionCodeGeneratorMIPS::GenerateTestAndBranch(HInstruction* instructi // The condition instruction has not been materialized, use its inputs as // the comparison and its condition as the branch condition. HCondition* condition = cond->AsCondition(); + Primitive::Type type = condition->InputAt(0)->GetType(); + LocationSummary* locations = cond->GetLocations(); + IfCondition if_cond = condition->GetCondition(); + MipsLabel* branch_target = true_target; - Register lhs = condition->GetLocations()->InAt(0).AsRegister<Register>(); - Location rhs_location = condition->GetLocations()->InAt(1); - Register rhs_reg = ZERO; - int32_t rhs_imm = 0; - bool use_imm = rhs_location.IsConstant(); - if (use_imm) { - rhs_imm = CodeGenerator::GetInt32ValueOf(rhs_location.GetConstant()); - } else { - rhs_reg = rhs_location.AsRegister<Register>(); - } - - IfCondition if_cond; - MipsLabel* non_fallthrough_target; if (true_target == nullptr) { if_cond = condition->GetOppositeCondition(); - non_fallthrough_target = false_target; - } else { - if_cond = condition->GetCondition(); - non_fallthrough_target = true_target; + branch_target = false_target; } - if (use_imm && rhs_imm == 0) { - switch (if_cond) { - case kCondEQ: - __ Beqz(lhs, non_fallthrough_target); - break; - case kCondNE: - __ Bnez(lhs, non_fallthrough_target); - break; - case kCondLT: - __ Bltz(lhs, non_fallthrough_target); - break; - case kCondGE: - __ Bgez(lhs, non_fallthrough_target); - break; - case kCondLE: - __ Blez(lhs, non_fallthrough_target); - break; - case kCondGT: - __ Bgtz(lhs, non_fallthrough_target); - break; - case kCondB: - break; // always false - case kCondBE: - __ Beqz(lhs, non_fallthrough_target); // <= 0 if zero - break; - case kCondA: - __ Bnez(lhs, non_fallthrough_target); // > 0 if non-zero - break; - case kCondAE: - __ B(non_fallthrough_target); // always true - break; - } - } else { - if (use_imm) { - // TODO: more efficient comparison with 16-bit constants without loading them into TMP. - rhs_reg = TMP; - __ LoadConst32(rhs_reg, rhs_imm); - } - switch (if_cond) { - case kCondEQ: - __ Beq(lhs, rhs_reg, non_fallthrough_target); - break; - case kCondNE: - __ Bne(lhs, rhs_reg, non_fallthrough_target); - break; - case kCondLT: - __ Blt(lhs, rhs_reg, non_fallthrough_target); - break; - case kCondGE: - __ Bge(lhs, rhs_reg, non_fallthrough_target); - break; - case kCondLE: - __ Bge(rhs_reg, lhs, non_fallthrough_target); - break; - case kCondGT: - __ Blt(rhs_reg, lhs, non_fallthrough_target); - break; - case kCondB: - __ Bltu(lhs, rhs_reg, non_fallthrough_target); - break; - case kCondAE: - __ Bgeu(lhs, rhs_reg, non_fallthrough_target); - break; - case kCondBE: - __ Bgeu(rhs_reg, lhs, non_fallthrough_target); - break; - case kCondA: - __ Bltu(rhs_reg, lhs, non_fallthrough_target); - break; - } + switch (type) { + default: + GenerateIntCompareAndBranch(if_cond, locations, branch_target); + break; + case Primitive::kPrimLong: + GenerateLongCompareAndBranch(if_cond, locations, branch_target); + break; + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + GenerateFpCompareAndBranch(if_cond, condition->IsGtBias(), type, locations, branch_target); + break; } } @@ -3413,9 +3902,9 @@ void InstructionCodeGeneratorMIPS::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNU } void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = load->IsInDexCache() + ? LocationSummary::kNoCall + : LocationSummary::kCallOnSlowPath; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::RequiresRegister()); @@ -4408,31 +4897,19 @@ void InstructionCodeGeneratorMIPS::VisitPackedSwitch(HPackedSwitch* switch_instr HBasicBlock* default_block = switch_instr->GetDefaultBlock(); // Create a set of compare/jumps. - Register temp_reg = TMP; - __ Addiu32(temp_reg, value_reg, -lower_bound); - // Jump to default if index is negative - // Note: We don't check the case that index is positive while value < lower_bound, because in - // this case, index >= num_entries must be true. So that we can save one branch instruction. - __ Bltz(temp_reg, codegen_->GetLabelOf(default_block)); - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - // Jump to successors[0] if value == lower_bound. - __ Beqz(temp_reg, codegen_->GetLabelOf(successors[0])); - int32_t last_index = 0; - for (; num_entries - last_index > 2; last_index += 2) { - __ Addiu(temp_reg, temp_reg, -2); - // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. - __ Bltz(temp_reg, codegen_->GetLabelOf(successors[last_index + 1])); - // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. - __ Beqz(temp_reg, codegen_->GetLabelOf(successors[last_index + 2])); - } - if (num_entries - last_index == 2) { - // The last missing case_value. - __ Addiu(temp_reg, temp_reg, -1); - __ Beqz(temp_reg, codegen_->GetLabelOf(successors[last_index + 1])); - } - - // And the default for any other value. + for (int32_t i = 0; i < num_entries; ++i) { + int32_t case_value = lower_bound + i; + MipsLabel* successor_label = codegen_->GetLabelOf(successors[i]); + if (case_value == 0) { + __ Beqz(value_reg, successor_label); + } else { + __ LoadConst32(TMP, case_value); + __ Beq(value_reg, TMP, successor_label); + } + } + + // Insert the default branch for every other value. if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { __ B(codegen_->GetLabelOf(default_block)); } diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index caf3174455..1ee6bdef8e 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -225,6 +225,18 @@ class InstructionCodeGeneratorMIPS : public HGraphVisitor { void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info, uint32_t dex_pc); void GenerateImplicitNullCheck(HNullCheck* instruction); void GenerateExplicitNullCheck(HNullCheck* instruction); + void GenerateIntCompare(IfCondition cond, LocationSummary* locations); + void GenerateIntCompareAndBranch(IfCondition cond, + LocationSummary* locations, + MipsLabel* label); + void GenerateLongCompareAndBranch(IfCondition cond, + LocationSummary* locations, + MipsLabel* label); + void GenerateFpCompareAndBranch(IfCondition cond, + bool gt_bias, + Primitive::Type type, + LocationSummary* locations, + MipsLabel* label); void GenerateTestAndBranch(HInstruction* instruction, size_t condition_input_index, MipsLabel* true_target, diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 1a9de15c6f..99f58dd2c5 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -3051,7 +3051,8 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) { } else { __ LoadFromOffset(kLoadDoubleword, out, current_method, ArtMethod::DexCacheResolvedTypesOffset(kMips64PointerSize).Int32Value()); - __ LoadFromOffset(kLoadUnsignedWord, out, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); + __ LoadFromOffset( + kLoadUnsignedWord, out, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); // TODO: We will need a read barrier here. if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); @@ -3105,9 +3106,9 @@ void InstructionCodeGeneratorMIPS64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_U } void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = load->IsInDexCache() + ? LocationSummary::kNoCall + : LocationSummary::kCallOnSlowPath; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::RequiresRegister()); @@ -3120,7 +3121,8 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) { __ LoadFromOffset(kLoadUnsignedWord, out, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); __ LoadFromOffset(kLoadDoubleword, out, out, mirror::Class::DexCacheStringsOffset().Int32Value()); - __ LoadFromOffset(kLoadUnsignedWord, out, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + __ LoadFromOffset( + kLoadUnsignedWord, out, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); // TODO: We will need a read barrier here. if (!load->IsInDexCache()) { @@ -3989,34 +3991,17 @@ void InstructionCodeGeneratorMIPS64::VisitPackedSwitch(HPackedSwitch* switch_ins GpuRegister value_reg = locations->InAt(0).AsRegister<GpuRegister>(); HBasicBlock* default_block = switch_instr->GetDefaultBlock(); - // Create a set of compare/jumps. - GpuRegister temp_reg = TMP; - if (IsInt<16>(-lower_bound)) { - __ Addiu(temp_reg, value_reg, -lower_bound); - } else { - __ LoadConst32(AT, -lower_bound); - __ Addu(temp_reg, value_reg, AT); - } - // Jump to default if index is negative - // Note: We don't check the case that index is positive while value < lower_bound, because in - // this case, index >= num_entries must be true. So that we can save one branch instruction. - __ Bltzc(temp_reg, codegen_->GetLabelOf(default_block)); - + // Create a series of compare/jumps. const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - // Jump to successors[0] if value == lower_bound. - __ Beqzc(temp_reg, codegen_->GetLabelOf(successors[0])); - int32_t last_index = 0; - for (; num_entries - last_index > 2; last_index += 2) { - __ Addiu(temp_reg, temp_reg, -2); - // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. - __ Bltzc(temp_reg, codegen_->GetLabelOf(successors[last_index + 1])); - // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. - __ Beqzc(temp_reg, codegen_->GetLabelOf(successors[last_index + 2])); - } - if (num_entries - last_index == 2) { - // The last missing case_value. - __ Addiu(temp_reg, temp_reg, -1); - __ Beqzc(temp_reg, codegen_->GetLabelOf(successors[last_index + 1])); + for (int32_t i = 0; i < num_entries; i++) { + int32_t case_value = lower_bound + i; + Mips64Label* succ = codegen_->GetLabelOf(successors[i]); + if (case_value == 0) { + __ Beqzc(value_reg, succ); + } else { + __ LoadConst32(TMP, case_value); + __ Beqc(value_reg, TMP, succ); + } } // And the default for any other value. diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 0fb552a59b..bc3256ec8c 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -42,6 +42,7 @@ namespace x86 { static constexpr int kCurrentMethodStackOffset = 0; static constexpr Register kMethodRegisterArgument = EAX; + static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI }; static constexpr int kC2ConditionMask = 0x400; @@ -433,6 +434,56 @@ class ArraySetSlowPathX86 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86); }; +// Slow path marking an object during a read barrier. +class ReadBarrierMarkSlowPathX86 : public SlowPathCode { + public: + ReadBarrierMarkSlowPathX86(HInstruction* instruction, Location out, Location obj) + : instruction_(instruction), out_(out), obj_(obj) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathX86"; } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + Register reg_out = out_.AsRegister<Register>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsLoadClass() || + instruction_->IsLoadString() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast()) + << "Unexpected instruction in read barrier marking slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + InvokeRuntimeCallingConvention calling_convention; + CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen); + x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark), + instruction_, + instruction_->GetDexPc(), + this); + CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>(); + x86_codegen->Move32(out_, Location::RegisterLocation(EAX)); + + RestoreLiveRegisters(codegen, locations); + __ jmp(GetExitLabel()); + } + + private: + HInstruction* const instruction_; + const Location out_; + const Location obj_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathX86); +}; + // Slow path generating a read barrier for a heap reference. class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode { public: @@ -454,7 +505,7 @@ class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode { // to be instrumented, e.g.: // // __ movl(out, Address(out, offset)); - // codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset); + // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); // // In that case, we have lost the information about the original // object, and the emitted read barrier cannot work properly. @@ -470,7 +521,9 @@ class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode { DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); DCHECK(!instruction_->IsInvoke() || (instruction_->IsInvokeStaticOrDirect() && - instruction_->GetLocations()->Intrinsified())); + instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier for heap reference slow path: " + << instruction_->DebugName(); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); @@ -612,14 +665,18 @@ class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode { class ReadBarrierForRootSlowPathX86 : public SlowPathCode { public: ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root) - : instruction_(instruction), out_(out), root_(root) {} + : instruction_(instruction), out_(out), root_(root) { + DCHECK(kEmitCompilerReadBarrier); + } void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); Register reg_out = out_.AsRegister<Register>(); DCHECK(locations->CanCall()); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); - DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()); + DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) + << "Unexpected instruction in read barrier for GC root slow path: " + << instruction_->DebugName(); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); @@ -1831,7 +1888,7 @@ void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { } void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { - GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); + codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); } void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) { @@ -4092,7 +4149,7 @@ void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { LOG(FATAL) << "Unreachable"; } -void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) { +void CodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) { /* * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence. * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model. @@ -4346,9 +4403,14 @@ void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldI if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) { // Long values can be loaded atomically into an XMM using movsd. - // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM - // and then copy the XMM into the output 32bits at a time). + // So we use an XMM register as a temp to achieve atomicity (first + // load the temp into the XMM and then copy the XMM into the + // output, 32 bits at a time). locations->AddTemp(Location::RequiresFpuRegister()); + } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); } } @@ -4386,9 +4448,32 @@ void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction, } case Primitive::kPrimInt: - case Primitive::kPrimNot: { __ movl(out.AsRegister<Register>(), Address(base, offset)); break; + + case Primitive::kPrimNot: { + // /* HeapReference<Object> */ out = *(base + offset) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp_loc = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier call. + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, base, offset, temp_loc, /* needs_null_check */ true); + if (is_volatile) { + codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + } else { + __ movl(out.AsRegister<Register>(), Address(base, offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + if (is_volatile) { + codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); + } + break; } case Primitive::kPrimLong: { @@ -4423,17 +4508,20 @@ void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction, UNREACHABLE(); } - // Longs are handled in the switch. - if (field_type != Primitive::kPrimLong) { + if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimLong) { + // Potential implicit null checks, in the case of reference or + // long fields, are handled in the previous switch statement. + } else { codegen_->MaybeRecordImplicitNullCheck(instruction); } if (is_volatile) { - GenerateMemoryBarrier(MemBarrierKind::kLoadAny); - } - - if (field_type == Primitive::kPrimNot) { - codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset); + if (field_type == Primitive::kPrimNot) { + // Memory barriers, in the case of references, are also handled + // in the previous switch statement. + } else { + codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } } } @@ -4498,7 +4586,7 @@ void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction, CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); if (is_volatile) { - GenerateMemoryBarrier(MemBarrierKind::kAnyStore); + codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); } bool maybe_record_implicit_null_check_done = false; @@ -4603,7 +4691,7 @@ void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction, } if (is_volatile) { - GenerateMemoryBarrier(MemBarrierKind::kAnyAny); + codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); } } @@ -4784,6 +4872,11 @@ void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) { Location::kOutputOverlap : Location::kNoOutputOverlap); } + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier. + if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { + locations->AddTemp(Location::RequiresRegister()); + } } void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { @@ -4791,12 +4884,13 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { Location obj_loc = locations->InAt(0); Register obj = obj_loc.AsRegister<Register>(); Location index = locations->InAt(1); + Location out_loc = locations->Out(); Primitive::Type type = instruction->GetType(); switch (type) { case Primitive::kPrimBoolean: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value(); - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { __ movzxb(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset)); @@ -4808,7 +4902,7 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimByte: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value(); - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { __ movsxb(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset)); @@ -4820,7 +4914,7 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimShort: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value(); - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { __ movsxw(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset)); @@ -4832,7 +4926,7 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimChar: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value(); - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { __ movzxw(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset)); @@ -4842,13 +4936,9 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { break; } - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + case Primitive::kPrimInt: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { __ movl(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset)); @@ -4858,20 +4948,56 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { break; } + case Primitive::kPrimNot: { + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + // /* HeapReference<Object> */ out = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier call. + codegen_->GenerateArrayLoadWithBakerReadBarrier( + instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); + } else { + Register out = out_loc.AsRegister<Register>(); + if (index.IsConstant()) { + uint32_t offset = + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + __ movl(out, Address(obj, offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); + } else { + __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow( + instruction, out_loc, out_loc, obj_loc, data_offset, index); + } + } + break; + } + case Primitive::kPrimLong: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value(); - Location out = locations->Out(); - DCHECK_NE(obj, out.AsRegisterPairLow<Register>()); + DCHECK_NE(obj, out_loc.AsRegisterPairLow<Register>()); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset)); + __ movl(out_loc.AsRegisterPairLow<Register>(), Address(obj, offset)); codegen_->MaybeRecordImplicitNullCheck(instruction); - __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize)); + __ movl(out_loc.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize)); } else { - __ movl(out.AsRegisterPairLow<Register>(), + __ movl(out_loc.AsRegisterPairLow<Register>(), Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset)); codegen_->MaybeRecordImplicitNullCheck(instruction); - __ movl(out.AsRegisterPairHigh<Register>(), + __ movl(out_loc.AsRegisterPairHigh<Register>(), Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize)); } break; @@ -4879,7 +5005,7 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimFloat: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value(); - XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); if (index.IsConstant()) { __ movss(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset)); @@ -4891,7 +5017,7 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimDouble: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value(); - XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); if (index.IsConstant()) { __ movsd(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset)); @@ -4906,23 +5032,12 @@ void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { UNREACHABLE(); } - if (type != Primitive::kPrimLong) { + if (type == Primitive::kPrimNot || type == Primitive::kPrimLong) { + // Potential implicit null checks, in the case of reference or + // long arrays, are handled in the previous switch statement. + } else { codegen_->MaybeRecordImplicitNullCheck(instruction); } - - if (type == Primitive::kPrimNot) { - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); - Location out = locations->Out(); - if (index.IsConstant()) { - uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset); - } else { - codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index); - } - } } void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) { @@ -5054,12 +5169,12 @@ void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) { // __ movl(temp2, temp); // // /* HeapReference<Class> */ temp = temp->component_type_ // __ movl(temp, Address(temp, component_offset)); - // codegen_->GenerateReadBarrier( + // codegen_->GenerateReadBarrierSlow( // instruction, temp_loc, temp_loc, temp2_loc, component_offset); // // // /* HeapReference<Class> */ temp2 = register_value->klass_ // __ movl(temp2, Address(register_value, class_offset)); - // codegen_->GenerateReadBarrier( + // codegen_->GenerateReadBarrierSlow( // instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc); // // __ cmpl(temp, temp2); @@ -5340,8 +5455,8 @@ void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instructio DCHECK_EQ(slow_path->GetSuccessor(), successor); } - __ fs()->cmpw(Address::Absolute( - Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0)); + __ fs()->cmpw(Address::Absolute(Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), + Immediate(0)); if (successor == nullptr) { __ j(kNotEqual, slow_path->GetEntryLabel()); __ Bind(slow_path->GetReturnLabel()); @@ -5622,32 +5737,16 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) { if (cls->IsReferrersClass()) { DCHECK(!cls->CanCallRuntime()); DCHECK(!cls->MustGenerateClinitCheck()); - uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value(); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_) - __ leal(out, Address(current_method, declaring_class_offset)); - // /* mirror::Class* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc); - } else { - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - __ movl(out, Address(current_method, declaring_class_offset)); - } + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); } else { // /* GcRoot<mirror::Class>[] */ out = // current_method.ptr_sized_fields_->dex_cache_resolved_types_ __ movl(out, Address(current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value())); - - size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex()); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::Class>* */ out = &out[type_index] - __ leal(out, Address(out, cache_offset)); - // /* mirror::Class* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc); - } else { - // /* GcRoot<mirror::Class> */ out = out[type_index] - __ movl(out, Address(out, cache_offset)); - } + // /* GcRoot<mirror::Class> */ out = out[type_index] + GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); @@ -5711,30 +5810,14 @@ void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) { Register out = out_loc.AsRegister<Register>(); Register current_method = locations->InAt(0).AsRegister<Register>(); - uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value(); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_) - __ leal(out, Address(current_method, declaring_class_offset)); - // /* mirror::Class* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc); - } else { - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - __ movl(out, Address(current_method, declaring_class_offset)); - } - + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value())); - - size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex()); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::String>* */ out = &out[string_index] - __ leal(out, Address(out, cache_offset)); - // /* mirror::String* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc); - } else { - // /* GcRoot<mirror::String> */ out = out[string_index] - __ movl(out, Address(out, cache_offset)); - } + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load); @@ -5782,6 +5865,14 @@ void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) { CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); } +static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { + return kEmitCompilerReadBarrier && + (kUseBakerReadBarrier || + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck); +} + void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); @@ -5807,21 +5898,22 @@ void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) { locations->SetOut(Location::RequiresRegister()); // When read barriers are enabled, we need a temporary register for // some cases. - if (kEmitCompilerReadBarrier && - (type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + if (TypeCheckNeedsATemporary(type_check_kind)) { locations->AddTemp(Location::RequiresRegister()); } } void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); Register obj = obj_loc.AsRegister<Register>(); Location cls = locations->InAt(1); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); + Location temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? + locations->GetTemp(0) : + Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -5837,10 +5929,9 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { } // /* HeapReference<Class> */ out = obj->klass_ - __ movl(out, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, temp_loc); - switch (instruction->GetTypeCheckKind()) { + switch (type_check_kind) { case TypeCheckKind::kExactCheck: { if (cls.IsRegister()) { __ cmpl(out, cls.AsRegister<Register>()); @@ -5861,17 +5952,8 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { // object to avoid doing a comparison we know will fail. NearLabel loop; __ Bind(&loop); - Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `out` into `temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - Register temp = temp_loc.AsRegister<Register>(); - __ movl(temp, out); - } // /* HeapReference<Class> */ out = out->super_class_ - __ movl(out, Address(out, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, temp_loc); __ testl(out, out); // If `out` is null, we use it for the result, and jump to `done`. __ j(kEqual, &done); @@ -5900,17 +5982,8 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { __ cmpl(out, Address(ESP, cls.GetStackIndex())); } __ j(kEqual, &success); - Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `out` into `temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - Register temp = temp_loc.AsRegister<Register>(); - __ movl(temp, out); - } // /* HeapReference<Class> */ out = out->super_class_ - __ movl(out, Address(out, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, temp_loc); __ testl(out, out); __ j(kNotEqual, &loop); // If `out` is null, we use it for the result, and jump to `done`. @@ -5934,17 +6007,8 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { } __ j(kEqual, &exact_check); // Otherwise, we need to check that the object's class is a non-primitive array. - Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `out` into `temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - Register temp = temp_loc.AsRegister<Register>(); - __ movl(temp, out); - } // /* HeapReference<Class> */ out = out->component_type_ - __ movl(out, Address(out, component_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset); + GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, temp_loc); __ testl(out, out); // If `out` is null, we use it for the result, and jump to `done`. __ j(kEqual, &done); @@ -5988,6 +6052,13 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { // HInstanceOf instruction (following the runtime calling // convention), which might be cluttered by the potential first // read barrier emission at the beginning of this method. + // + // TODO: Introduce a new runtime entry point taking the object + // to test (instead of its class) as argument, and let it deal + // with the read barrier issues. This will let us refactor this + // case of the `switch` code as it was previously (with a direct + // call to the runtime not using a type checking slow path). + // This should also be beneficial for the other cases above. DCHECK(locations->OnlyCallsOnSlowPath()); slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction, /* is_fatal */ false); @@ -6040,27 +6111,27 @@ void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) { locations->AddTemp(Location::RequiresRegister()); // When read barriers are enabled, we need an additional temporary // register for some cases. - if (kEmitCompilerReadBarrier && - (type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + if (TypeCheckNeedsATemporary(type_check_kind)) { locations->AddTemp(Location::RequiresRegister()); } } void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); Register obj = obj_loc.AsRegister<Register>(); Location cls = locations->InAt(1); Location temp_loc = locations->GetTemp(0); Register temp = temp_loc.AsRegister<Register>(); + Location temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? + locations->GetTemp(1) : + Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); bool is_type_check_slow_path_fatal = (type_check_kind == TypeCheckKind::kExactCheck || type_check_kind == TypeCheckKind::kAbstractClassCheck || @@ -6080,8 +6151,7 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { } // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); switch (type_check_kind) { case TypeCheckKind::kExactCheck: @@ -6103,18 +6173,8 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { // object to avoid doing a comparison we know will fail. NearLabel loop, compare_classes; __ Bind(&loop); - Location temp2_loc = - kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `temp` into `temp2` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - Register temp2 = temp2_loc.AsRegister<Register>(); - __ movl(temp2, temp); - } // /* HeapReference<Class> */ temp = temp->super_class_ - __ movl(temp, Address(temp, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, temp2_loc); // If the class reference currently in `temp` is not null, jump // to the `compare_classes` label to compare it with the checked @@ -6127,8 +6187,7 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&compare_classes); @@ -6154,18 +6213,8 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { } __ j(kEqual, &done); - Location temp2_loc = - kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `temp` into `temp2` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - Register temp2 = temp2_loc.AsRegister<Register>(); - __ movl(temp2, temp); - } // /* HeapReference<Class> */ temp = temp->super_class_ - __ movl(temp, Address(temp, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, temp2_loc); // If the class reference currently in `temp` is not null, jump // back at the beginning of the loop. @@ -6177,8 +6226,7 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); break; } @@ -6195,19 +6243,8 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { __ j(kEqual, &done); // Otherwise, we need to check that the object's class is a non-primitive array. - Location temp2_loc = - kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `temp` into `temp2` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - Register temp2 = temp2_loc.AsRegister<Register>(); - __ movl(temp2, temp); - } // /* HeapReference<Class> */ temp = temp->component_type_ - __ movl(temp, Address(temp, component_offset)); - codegen_->MaybeGenerateReadBarrier( - instruction, temp_loc, temp_loc, temp2_loc, component_offset); + GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, temp2_loc); // If the component type is not null (i.e. the object is indeed // an array), jump to label `check_non_primitive_component_type` @@ -6221,8 +6258,7 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&check_non_primitive_component_type); @@ -6230,8 +6266,7 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { __ j(kEqual, &done); // Same comment as above regarding `temp` and the slow path. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); break; } @@ -6248,6 +6283,13 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { // instruction (following the runtime calling convention), which // might be cluttered by the potential first read barrier // emission at the beginning of this method. + // + // TODO: Introduce a new runtime entry point taking the object + // to test (instead of its class) as argument, and let it deal + // with the read barrier issues. This will let us refactor this + // case of the `switch` code as it was previously (with a direct + // call to the runtime not using a type checking slow path). + // This should also be beneficial for the other cases above. __ jmp(type_check_slow_path->GetEntryLabel()); break; } @@ -6409,14 +6451,226 @@ void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instr } } -void CodeGeneratorX86::GenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { +void InstructionCodeGeneratorX86::GenerateReferenceLoadOneRegister(HInstruction* instruction, + Location out, + uint32_t offset, + Location temp) { + Register out_reg = out.AsRegister<Register>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, out_reg, offset, temp, /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // Save the value of `out` into `temp` before overwriting it + // in the following move operation, as we will need it for the + // read barrier below. + __ movl(temp.AsRegister<Register>(), out_reg); + // /* HeapReference<Object> */ out = *(out + offset) + __ movl(out_reg, Address(out_reg, offset)); + codegen_->GenerateReadBarrierSlow(instruction, out, out, temp, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + __ movl(out_reg, Address(out_reg, offset)); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + +void InstructionCodeGeneratorX86::GenerateReferenceLoadTwoRegisters(HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location temp) { + Register out_reg = out.AsRegister<Register>(); + Register obj_reg = obj.AsRegister<Register>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, obj_reg, offset, temp, /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ movl(out_reg, Address(obj_reg, offset)); + codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ movl(out_reg, Address(obj_reg, offset)); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + +void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + Register obj, + uint32_t offset) { + Register root_reg = root.AsRegister<Register>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + // Fast path implementation of art::ReadBarrier::BarrierForRoot when + // Baker's read barrier are used: + // + // root = obj.field; + // if (Thread::Current()->GetIsGcMarking()) { + // root = ReadBarrier::Mark(root) + // } + + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + __ movl(root_reg, Address(obj, offset)); + static_assert( + sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), + "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " + "have different sizes."); + static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::CompressedReference<mirror::Object> and int32_t " + "have different sizes."); + + // Slow path used to mark the GC root `root`. + SlowPathCode* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, root, root); + codegen_->AddSlowPath(slow_path); + + __ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86WordSize>().Int32Value()), + Immediate(0)); + __ j(kNotEqual, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); + } else { + // GC root loaded through a slow path for read barriers other + // than Baker's. + // /* GcRoot<mirror::Object>* */ root = obj + offset + __ leal(root_reg, Address(obj, offset)); + // /* mirror::Object* */ root = root->Read() + codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); + } + } else { + // Plain GC root load with no read barrier. + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + __ movl(root_reg, Address(obj, offset)); + } +} + +void CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t offset, + Location temp, + bool needs_null_check) { DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + // /* HeapReference<Object> */ ref = *(obj + offset) + Address src(obj, offset); + GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check); +} + +void CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // /* HeapReference<Object> */ ref = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + Address src = index.IsConstant() ? + Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset) : + Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset); + GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check); +} + +void CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + const Address& src, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // In slow path based read barriers, the read barrier call is + // inserted after the original load. However, in fast path based + // Baker's read barriers, we need to perform the load of + // mirror::Object::monitor_ *before* the original reference load. + // This load-load ordering is required by the read barrier. + // The fast path/slow path (for Baker's algorithm) should look like: + // + // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); + // lfence; // Load fence or artificial data dependency to prevent load-load reordering + // HeapReference<Object> ref = *src; // Original reference load. + // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // if (is_gray) { + // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. + // } + // + // Note: the original implementation in ReadBarrier::Barrier is + // slightly more complex as: + // - it implements the load-load fence using a data dependency on + // the high-bits of rb_state, which are expected to be all zeroes; + // - it performs additional checks that we do not do here for + // performance reasons. + + Register ref_reg = ref.AsRegister<Register>(); + Register temp_reg = temp.AsRegister<Register>(); + uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); + + // /* int32_t */ monitor = obj->monitor_ + __ movl(temp_reg, Address(obj, monitor_offset)); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + // /* LockWord */ lock_word = LockWord(monitor) + static_assert(sizeof(LockWord) == sizeof(int32_t), + "art::LockWord and int32_t have different sizes."); + // /* uint32_t */ rb_state = lock_word.ReadBarrierState() + __ shrl(temp_reg, Immediate(LockWord::kReadBarrierStateShift)); + __ andl(temp_reg, Immediate(LockWord::kReadBarrierStateMask)); + static_assert( + LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_, + "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_."); + + // Load fence to prevent load-load reordering. + // Note that this is a no-op, thanks to the x86 memory model. + GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + + // The actual reference load. + // /* HeapReference<Object> */ ref = *src + __ movl(ref_reg, src); + + // Object* ref = ref_addr->AsMirrorPtr() + __ MaybeUnpoisonHeapReference(ref_reg); + + // Slow path used to mark the object `ref` when it is gray. + SlowPathCode* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, ref, ref); + AddSlowPath(slow_path); + + // if (rb_state == ReadBarrier::gray_ptr_) + // ref = ReadBarrier::Mark(ref); + __ cmpl(temp_reg, Immediate(ReadBarrier::gray_ptr_)); + __ j(kEqual, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + +void CodeGeneratorX86::GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { + DCHECK(kEmitCompilerReadBarrier); + + // Insert a slow path based read barrier *after* the reference load. + // // If heap poisoning is enabled, the unpoisoning of the loaded // reference will be carried out by the runtime within the slow // path. @@ -6430,57 +6684,41 @@ void CodeGeneratorX86::GenerateReadBarrier(HInstruction* instruction, ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index); AddSlowPath(slow_path); - // TODO: When read barrier has a fast path, add it here. - /* Currently the read barrier call is inserted after the original load. - * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the - * original load. This load-load ordering is required by the read barrier. - * The fast path/slow path (for Baker's algorithm) should look like: - * - * bool isGray = obj.LockWord & kReadBarrierMask; - * lfence; // load fence or artificial data dependence to prevent load-load reordering - * ref = obj.field; // this is the original load - * if (isGray) { - * ref = Mark(ref); // ideally the slow path just does Mark(ref) - * } - */ - __ jmp(slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } -void CodeGeneratorX86::MaybeGenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { +void CodeGeneratorX86::MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { if (kEmitCompilerReadBarrier) { + // Baker's read barriers shall be handled by the fast path + // (CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier). + DCHECK(!kUseBakerReadBarrier); // If heap poisoning is enabled, unpoisoning will be taken care of // by the runtime within the slow path. - GenerateReadBarrier(instruction, out, ref, obj, offset, index); + GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); } else if (kPoisonHeapReferences) { __ UnpoisonHeapReference(out.AsRegister<Register>()); } } -void CodeGeneratorX86::GenerateReadBarrierForRoot(HInstruction* instruction, - Location out, - Location root) { +void CodeGeneratorX86::GenerateReadBarrierForRootSlow(HInstruction* instruction, + Location out, + Location root) { DCHECK(kEmitCompilerReadBarrier); + // Insert a slow path based read barrier *after* the GC root load. + // // Note that GC roots are not affected by heap poisoning, so we do // not need to do anything special for this here. SlowPathCode* slow_path = new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root); AddSlowPath(slow_path); - // TODO: Implement a fast path for ReadBarrierForRoot, performing - // the following operation (for Baker's algorithm): - // - // if (thread.tls32_.is_gc_marking) { - // root = Mark(root); - // } - __ jmp(slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } @@ -6514,65 +6752,29 @@ void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) { locations->SetInAt(0, Location::RequiresRegister()); } -void InstructionCodeGeneratorX86::GenPackedSwitchWithCompares(Register value_reg, - int32_t lower_bound, - uint32_t num_entries, - HBasicBlock* switch_block, - HBasicBlock* default_block) { - // Figure out the correct compare values and jump conditions. - // Handle the first compare/branch as a special case because it might - // jump to the default case. - DCHECK_GT(num_entries, 2u); - Condition first_condition; - uint32_t index; - const ArenaVector<HBasicBlock*>& successors = switch_block->GetSuccessors(); - if (lower_bound != 0) { - first_condition = kLess; - __ cmpl(value_reg, Immediate(lower_bound)); - __ j(first_condition, codegen_->GetLabelOf(default_block)); - __ j(kEqual, codegen_->GetLabelOf(successors[0])); - - index = 1; - } else { - // Handle all the compare/jumps below. - first_condition = kBelow; - index = 0; - } - - // Handle the rest of the compare/jumps. - for (; index + 1 < num_entries; index += 2) { - int32_t compare_to_value = lower_bound + index + 1; - __ cmpl(value_reg, Immediate(compare_to_value)); - // Jump to successors[index] if value < case_value[index]. - __ j(first_condition, codegen_->GetLabelOf(successors[index])); - // Jump to successors[index + 1] if value == case_value[index + 1]. - __ j(kEqual, codegen_->GetLabelOf(successors[index + 1])); - } - - if (index != num_entries) { - // There are an odd number of entries. Handle the last one. - DCHECK_EQ(index + 1, num_entries); - __ cmpl(value_reg, Immediate(lower_bound + index)); - __ j(kEqual, codegen_->GetLabelOf(successors[index])); - } - - // And the default for any other value. - if (!codegen_->GoesToNextBlock(switch_block, default_block)) { - __ jmp(codegen_->GetLabelOf(default_block)); - } -} - void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) { int32_t lower_bound = switch_instr->GetStartValue(); - uint32_t num_entries = switch_instr->GetNumEntries(); + int32_t num_entries = switch_instr->GetNumEntries(); LocationSummary* locations = switch_instr->GetLocations(); Register value_reg = locations->InAt(0).AsRegister<Register>(); + HBasicBlock* default_block = switch_instr->GetDefaultBlock(); + + // Create a series of compare/jumps. + const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); + for (int i = 0; i < num_entries; i++) { + int32_t case_value = lower_bound + i; + if (case_value == 0) { + __ testl(value_reg, value_reg); + } else { + __ cmpl(value_reg, Immediate(case_value)); + } + __ j(kEqual, codegen_->GetLabelOf(successors[i])); + } - GenPackedSwitchWithCompares(value_reg, - lower_bound, - num_entries, - switch_instr->GetBlock(), - switch_instr->GetDefaultBlock()); + // And the default for any other value. + if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { + __ jmp(codegen_->GetLabelOf(default_block)); + } } void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) { @@ -6589,20 +6791,11 @@ void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) { void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) { int32_t lower_bound = switch_instr->GetStartValue(); - uint32_t num_entries = switch_instr->GetNumEntries(); + int32_t num_entries = switch_instr->GetNumEntries(); LocationSummary* locations = switch_instr->GetLocations(); Register value_reg = locations->InAt(0).AsRegister<Register>(); HBasicBlock* default_block = switch_instr->GetDefaultBlock(); - if (num_entries <= kPackedSwitchJumpTableThreshold) { - GenPackedSwitchWithCompares(value_reg, - lower_bound, - num_entries, - switch_instr->GetBlock(), - default_block); - return; - } - // Optimizing has a jump area. Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); Register constant_area = locations->InAt(1).AsRegister<Register>(); @@ -6614,7 +6807,7 @@ void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_ } // Is the value in range? - DCHECK_GE(num_entries, 1u); + DCHECK_GE(num_entries, 1); __ cmpl(value_reg, Immediate(num_entries - 1)); __ j(kAbove, codegen_->GetLabelOf(default_block)); @@ -6839,7 +7032,7 @@ Address CodeGeneratorX86::LiteralCaseTable(HX86PackedSwitch* switch_instr, // TODO: target as memory. void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) { if (!target.IsValid()) { - DCHECK(type == Primitive::kPrimVoid); + DCHECK_EQ(type, Primitive::kPrimVoid); return; } diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index f9403a67c0..7c292fa103 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -195,11 +195,6 @@ class InstructionCodeGeneratorX86 : public HGraphVisitor { X86Assembler* GetAssembler() const { return assembler_; } - // The compare/jump sequence will generate about (1.5 * num_entries) instructions. A jump - // table version generates 7 instructions and num_entries literals. Compare/jump sequence will - // generates less code/data with a small num_entries. - static constexpr uint32_t kPackedSwitchJumpTableThreshold = 5; - private: // Generate code for the given suspend check. If not null, `successor` // is the block to branch to if the suspend check is not needed, and after @@ -219,11 +214,44 @@ class InstructionCodeGeneratorX86 : public HGraphVisitor { void GenerateShlLong(const Location& loc, int shift); void GenerateShrLong(const Location& loc, int shift); void GenerateUShrLong(const Location& loc, int shift); - void GenerateMemoryBarrier(MemBarrierKind kind); + void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info, bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); + + // Generate a heap reference load using one register `out`: + // + // out <- *(out + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // Register `temp` is used when generating a read barrier. + void GenerateReferenceLoadOneRegister(HInstruction* instruction, + Location out, + uint32_t offset, + Location temp); + // Generate a heap reference load using two different registers + // `out` and `obj`: + // + // out <- *(obj + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // Register `temp` is used when generating a Baker's read barrier. + void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location temp); + // Generate a GC root reference load: + // + // root <- *(obj + offset) + // + // while honoring read barriers (if any). + void GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + Register obj, + uint32_t offset); + // Push value to FPU stack. `is_fp` specifies whether the value is floating point or not. // `is_wide` specifies whether it is long/double or not. void PushOntoFPStack(Location source, uint32_t temp_offset, @@ -241,11 +269,6 @@ class InstructionCodeGeneratorX86 : public HGraphVisitor { void GenerateFPJumps(HCondition* cond, Label* true_label, Label* false_label); void GenerateLongComparesAndJumps(HCondition* cond, Label* true_label, Label* false_label); void HandleGoto(HInstruction* got, HBasicBlock* successor); - void GenPackedSwitchWithCompares(Register value_reg, - int32_t lower_bound, - uint32_t num_entries, - HBasicBlock* switch_block, - HBasicBlock* default_block); X86Assembler* const assembler_; CodeGeneratorX86* const codegen_; @@ -364,6 +387,8 @@ class CodeGeneratorX86 : public CodeGenerator { Register value, bool value_can_be_null); + void GenerateMemoryBarrier(MemBarrierKind kind); + Label* GetLabelOf(HBasicBlock* block) const { return CommonGetLabelOf<Label>(block_labels_, block); } @@ -405,7 +430,26 @@ class CodeGeneratorX86 : public CodeGenerator { void Finalize(CodeAllocator* allocator) OVERRIDE; - // Generate a read barrier for a heap reference within `instruction`. + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference field load when Baker's read barriers are used. + void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location out, + Register obj, + uint32_t offset, + Location temp, + bool needs_null_check); + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference array load when Baker's read barriers are used. + void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location out, + Register obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check); + + // Generate a read barrier for a heap reference within `instruction` + // using a slow path. // // A read barrier for an object reference read from the heap is // implemented as a call to the artReadBarrierSlow runtime entry @@ -422,23 +466,25 @@ class CodeGeneratorX86 : public CodeGenerator { // When `index` is provided (i.e. for array accesses), the offset // value passed to artReadBarrierSlow is adjusted to take `index` // into account. - void GenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // If read barriers are enabled, generate a read barrier for a heap reference. - // If heap poisoning is enabled, also unpoison the reference in `out`. - void MaybeGenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // Generate a read barrier for a GC root within `instruction`. + void GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // If read barriers are enabled, generate a read barrier for a heap + // reference using a slow path. If heap poisoning is enabled, also + // unpoison the reference in `out`. + void MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // Generate a read barrier for a GC root within `instruction` using + // a slow path. // // A read barrier for an object reference GC root is implemented as // a call to the artReadBarrierForRootSlow runtime entry point, @@ -448,9 +494,18 @@ class CodeGeneratorX86 : public CodeGenerator { // // The `out` location contains the value returned by // artReadBarrierForRootSlow. - void GenerateReadBarrierForRoot(HInstruction* instruction, Location out, Location root); + void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); private: + // Factored implementation of GenerateFieldLoadWithBakerReadBarrier + // and GenerateArrayLoadWithBakerReadBarrier. + void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + const Address& src, + Location temp, + bool needs_null_check); + Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp); struct PcRelativeDexCacheAccessInfo { diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 610ce6741a..92cef5f226 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -41,10 +41,6 @@ namespace x86_64 { static constexpr int kCurrentMethodStackOffset = 0; static constexpr Register kMethodRegisterArgument = RDI; -// The compare/jump sequence will generate about (1.5 * num_entries) instructions. A jump -// table version generates 7 instructions and num_entries literals. Compare/jump sequence will -// generates less code/data with a small num_entries. -static constexpr uint32_t kPackedSwitchJumpTableThreshold = 5; static constexpr Register kCoreCalleeSaves[] = { RBX, RBP, R12, R13, R14, R15 }; static constexpr FloatRegister kFpuCalleeSaves[] = { XMM12, XMM13, XMM14, XMM15 }; @@ -456,6 +452,56 @@ class ArraySetSlowPathX86_64 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86_64); }; +// Slow path marking an object during a read barrier. +class ReadBarrierMarkSlowPathX86_64 : public SlowPathCode { + public: + ReadBarrierMarkSlowPathX86_64(HInstruction* instruction, Location out, Location obj) + : instruction_(instruction), out_(out), obj_(obj) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathX86_64"; } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + Register reg_out = out_.AsRegister<Register>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsLoadClass() || + instruction_->IsLoadString() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast()) + << "Unexpected instruction in read barrier marking slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + InvokeRuntimeCallingConvention calling_convention; + CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen); + x86_64_codegen->Move(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_); + x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark), + instruction_, + instruction_->GetDexPc(), + this); + CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>(); + x86_64_codegen->Move(out_, Location::RegisterLocation(RAX)); + + RestoreLiveRegisters(codegen, locations); + __ jmp(GetExitLabel()); + } + + private: + HInstruction* const instruction_; + const Location out_; + const Location obj_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathX86_64); +}; + // Slow path generating a read barrier for a heap reference. class ReadBarrierForHeapReferenceSlowPathX86_64 : public SlowPathCode { public: @@ -477,7 +523,7 @@ class ReadBarrierForHeapReferenceSlowPathX86_64 : public SlowPathCode { // reference load to be instrumented, e.g.: // // __ movl(out, Address(out, offset)); - // codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset); + // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); // // In that case, we have lost the information about the original // object, and the emitted read barrier cannot work properly. @@ -493,7 +539,9 @@ class ReadBarrierForHeapReferenceSlowPathX86_64 : public SlowPathCode { DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.AsRegister())) << out_; DCHECK(!instruction_->IsInvoke() || (instruction_->IsInvokeStaticOrDirect() && - instruction_->GetLocations()->Intrinsified())); + instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier for heap reference slow path: " + << instruction_->DebugName(); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); @@ -634,13 +682,17 @@ class ReadBarrierForHeapReferenceSlowPathX86_64 : public SlowPathCode { class ReadBarrierForRootSlowPathX86_64 : public SlowPathCode { public: ReadBarrierForRootSlowPathX86_64(HInstruction* instruction, Location out, Location root) - : instruction_(instruction), out_(out), root_(root) {} + : instruction_(instruction), out_(out), root_(root) { + DCHECK(kEmitCompilerReadBarrier); + } void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); DCHECK(locations->CanCall()); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg())); - DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()); + DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) + << "Unexpected instruction in read barrier for GC root slow path: " + << instruction_->DebugName(); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); @@ -731,7 +783,7 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: // temp = thread->string_init_entrypoint __ gs()->movl(temp.AsRegister<CpuRegister>(), - Address::Absolute(invoke->GetStringInitOffset(), true)); + Address::Absolute(invoke->GetStringInitOffset(), /* no_rip */ true)); break; case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); @@ -748,7 +800,7 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, invoke->GetDexCacheArrayOffset()); __ movq(temp.AsRegister<CpuRegister>(), - Address::Absolute(kDummy32BitOffset, false /* no_rip */)); + Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); // Bind the label at the end of the "movl" insn. __ Bind(&pc_relative_dex_cache_patches_.back().label); break; @@ -907,7 +959,7 @@ void CodeGeneratorX86_64::InvokeRuntime(int32_t entry_point_offset, uint32_t dex_pc, SlowPathCode* slow_path) { ValidateInvokeRuntime(instruction, slow_path); - __ gs()->call(Address::Absolute(entry_point_offset, true)); + __ gs()->call(Address::Absolute(entry_point_offset, /* no_rip */ true)); RecordPcInfo(instruction, dex_pc, slow_path); } @@ -1939,7 +1991,7 @@ void LocationsBuilderX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) } void InstructionCodeGeneratorX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { - GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); + codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); } void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) { @@ -2667,7 +2719,8 @@ void InstructionCodeGeneratorX86_64::VisitTypeConversion(HTypeConversion* conver } else { DCHECK(in.GetConstant()->IsIntConstant()); __ movl(out.AsRegister<CpuRegister>(), - Immediate(static_cast<uint16_t>(in.GetConstant()->AsIntConstant()->GetValue()))); + Immediate(static_cast<uint16_t>( + in.GetConstant()->AsIntConstant()->GetValue()))); } break; @@ -2911,7 +2964,8 @@ void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) { __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ addss(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue())); + codegen_->LiteralFloatAddress( + second.GetConstant()->AsFloatConstant()->GetValue())); } else { DCHECK(second.IsStackSlot()); __ addss(first.AsFpuRegister<XmmRegister>(), @@ -2925,7 +2979,8 @@ void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) { __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ addsd(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue())); + codegen_->LiteralDoubleAddress( + second.GetConstant()->AsDoubleConstant()->GetValue())); } else { DCHECK(second.IsDoubleStackSlot()); __ addsd(first.AsFpuRegister<XmmRegister>(), @@ -3000,7 +3055,8 @@ void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) { __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ subss(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue())); + codegen_->LiteralFloatAddress( + second.GetConstant()->AsFloatConstant()->GetValue())); } else { DCHECK(second.IsStackSlot()); __ subss(first.AsFpuRegister<XmmRegister>(), @@ -3014,7 +3070,8 @@ void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) { __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ subsd(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue())); + codegen_->LiteralDoubleAddress( + second.GetConstant()->AsDoubleConstant()->GetValue())); } else { DCHECK(second.IsDoubleStackSlot()); __ subsd(first.AsFpuRegister<XmmRegister>(), @@ -3121,7 +3178,8 @@ void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) { __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ mulss(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue())); + codegen_->LiteralFloatAddress( + second.GetConstant()->AsFloatConstant()->GetValue())); } else { DCHECK(second.IsStackSlot()); __ mulss(first.AsFpuRegister<XmmRegister>(), @@ -3136,7 +3194,8 @@ void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) { __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ mulsd(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue())); + codegen_->LiteralDoubleAddress( + second.GetConstant()->AsDoubleConstant()->GetValue())); } else { DCHECK(second.IsDoubleStackSlot()); __ mulsd(first.AsFpuRegister<XmmRegister>(), @@ -3542,7 +3601,8 @@ void InstructionCodeGeneratorX86_64::VisitDiv(HDiv* div) { __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ divss(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue())); + codegen_->LiteralFloatAddress( + second.GetConstant()->AsFloatConstant()->GetValue())); } else { DCHECK(second.IsStackSlot()); __ divss(first.AsFpuRegister<XmmRegister>(), @@ -3556,7 +3616,8 @@ void InstructionCodeGeneratorX86_64::VisitDiv(HDiv* div) { __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>()); } else if (second.IsConstant()) { __ divsd(first.AsFpuRegister<XmmRegister>(), - codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue())); + codegen_->LiteralDoubleAddress( + second.GetConstant()->AsDoubleConstant()->GetValue())); } else { DCHECK(second.IsDoubleStackSlot()); __ divsd(first.AsFpuRegister<XmmRegister>(), @@ -3755,6 +3816,7 @@ void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) { } default: LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); + UNREACHABLE(); } } @@ -3959,10 +4021,10 @@ void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED LOG(FATAL) << "Unimplemented"; } -void InstructionCodeGeneratorX86_64::GenerateMemoryBarrier(MemBarrierKind kind) { +void CodeGeneratorX86_64::GenerateMemoryBarrier(MemBarrierKind kind) { /* * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence. - * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model. + * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86-64 memory model. * For those cases, all we need to ensure is that there is a scheduling barrier in place. */ switch (kind) { @@ -4002,6 +4064,11 @@ void LocationsBuilderX86_64::HandleFieldGet(HInstruction* instruction) { Location::RequiresRegister(), object_field_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); } + if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorX86_64::GenerateFieldLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); + } } void InstructionCodeGeneratorX86_64::HandleFieldGet(HInstruction* instruction, @@ -4037,12 +4104,36 @@ void InstructionCodeGeneratorX86_64::HandleFieldGet(HInstruction* instruction, break; } - case Primitive::kPrimInt: - case Primitive::kPrimNot: { + case Primitive::kPrimInt: { __ movl(out.AsRegister<CpuRegister>(), Address(base, offset)); break; } + case Primitive::kPrimNot: { + // /* HeapReference<Object> */ out = *(base + offset) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp_loc = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier call. + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, base, offset, temp_loc, /* needs_null_check */ true); + if (is_volatile) { + codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + } else { + __ movl(out.AsRegister<CpuRegister>(), Address(base, offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + if (is_volatile) { + codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); + } + break; + } + case Primitive::kPrimLong: { __ movq(out.AsRegister<CpuRegister>(), Address(base, offset)); break; @@ -4063,14 +4154,20 @@ void InstructionCodeGeneratorX86_64::HandleFieldGet(HInstruction* instruction, UNREACHABLE(); } - codegen_->MaybeRecordImplicitNullCheck(instruction); - - if (is_volatile) { - GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + if (field_type == Primitive::kPrimNot) { + // Potential implicit null checks, in the case of reference + // fields, are handled in the previous switch statement. + } else { + codegen_->MaybeRecordImplicitNullCheck(instruction); } - if (field_type == Primitive::kPrimNot) { - codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset); + if (is_volatile) { + if (field_type == Primitive::kPrimNot) { + // Memory barriers, in the case of references, are also handled + // in the previous switch statement. + } else { + codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } } } @@ -4124,7 +4221,7 @@ void InstructionCodeGeneratorX86_64::HandleFieldSet(HInstruction* instruction, uint32_t offset = field_info.GetFieldOffset().Uint32Value(); if (is_volatile) { - GenerateMemoryBarrier(MemBarrierKind::kAnyStore); + codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); } bool maybe_record_implicit_null_check_done = false; @@ -4230,7 +4327,7 @@ void InstructionCodeGeneratorX86_64::HandleFieldSet(HInstruction* instruction, } if (is_volatile) { - GenerateMemoryBarrier(MemBarrierKind::kAnyAny); + codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); } } @@ -4407,6 +4504,11 @@ void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) { Location::RequiresRegister(), object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); } + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorX86_64::GenerateArrayLoadWithBakerReadBarrier. + if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { + locations->AddTemp(Location::RequiresRegister()); + } } void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { @@ -4414,12 +4516,13 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { Location obj_loc = locations->InAt(0); CpuRegister obj = obj_loc.AsRegister<CpuRegister>(); Location index = locations->InAt(1); - Primitive::Type type = instruction->GetType(); + Location out_loc = locations->Out(); + Primitive::Type type = instruction->GetType(); switch (type) { case Primitive::kPrimBoolean: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value(); - CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister out = out_loc.AsRegister<CpuRegister>(); if (index.IsConstant()) { __ movzxb(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset)); @@ -4431,7 +4534,7 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimByte: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value(); - CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister out = out_loc.AsRegister<CpuRegister>(); if (index.IsConstant()) { __ movsxb(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset)); @@ -4443,7 +4546,7 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimShort: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value(); - CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister out = out_loc.AsRegister<CpuRegister>(); if (index.IsConstant()) { __ movsxw(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset)); @@ -4455,7 +4558,7 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimChar: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value(); - CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister out = out_loc.AsRegister<CpuRegister>(); if (index.IsConstant()) { __ movzxw(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset)); @@ -4465,13 +4568,9 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { break; } - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + case Primitive::kPrimInt: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); - CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister out = out_loc.AsRegister<CpuRegister>(); if (index.IsConstant()) { __ movl(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset)); @@ -4481,9 +4580,46 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { break; } + case Primitive::kPrimNot: { + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + // /* HeapReference<Object> */ out = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier call. + codegen_->GenerateArrayLoadWithBakerReadBarrier( + instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); + } else { + CpuRegister out = out_loc.AsRegister<CpuRegister>(); + if (index.IsConstant()) { + uint32_t offset = + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + __ movl(out, Address(obj, offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); + } else { + __ movl(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow( + instruction, out_loc, out_loc, obj_loc, data_offset, index); + } + } + break; + } + case Primitive::kPrimLong: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value(); - CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister out = out_loc.AsRegister<CpuRegister>(); if (index.IsConstant()) { __ movq(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset)); @@ -4495,7 +4631,7 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimFloat: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value(); - XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); if (index.IsConstant()) { __ movss(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset)); @@ -4507,7 +4643,7 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimDouble: { uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value(); - XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); if (index.IsConstant()) { __ movsd(out, Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset)); @@ -4521,20 +4657,12 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { LOG(FATAL) << "Unreachable type " << type; UNREACHABLE(); } - codegen_->MaybeRecordImplicitNullCheck(instruction); if (type == Primitive::kPrimNot) { - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); - Location out = locations->Out(); - if (index.IsConstant()) { - uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset); - } else { - codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index); - } + // Potential implicit null checks, in the case of reference + // arrays, are handled in the previous switch statement. + } else { + codegen_->MaybeRecordImplicitNullCheck(instruction); } } @@ -4658,12 +4786,12 @@ void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) { // __ movl(temp2, temp); // // /* HeapReference<Class> */ temp = temp->component_type_ // __ movl(temp, Address(temp, component_offset)); - // codegen_->GenerateReadBarrier( + // codegen_->GenerateReadBarrierSlow( // instruction, temp_loc, temp_loc, temp2_loc, component_offset); // // // /* HeapReference<Class> */ temp2 = register_value->klass_ // __ movl(temp2, Address(register_value, class_offset)); - // codegen_->GenerateReadBarrier( + // codegen_->GenerateReadBarrierSlow( // instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc); // // __ cmpl(temp, temp2); @@ -4889,8 +5017,8 @@ void CodeGeneratorX86_64::MarkGCCard(CpuRegister temp, __ testl(value, value); __ j(kEqual, &is_null); } - __ gs()->movq(card, Address::Absolute( - Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true)); + __ gs()->movq(card, Address::Absolute(Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), + /* no_rip */ true)); __ movq(temp, object); __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift)); __ movb(Address(temp, card, TIMES_1, 0), card); @@ -4949,8 +5077,9 @@ void InstructionCodeGeneratorX86_64::GenerateSuspendCheck(HSuspendCheck* instruc DCHECK_EQ(slow_path->GetSuccessor(), successor); } - __ gs()->cmpw(Address::Absolute( - Thread::ThreadFlagsOffset<kX86_64WordSize>().Int32Value(), true), Immediate(0)); + __ gs()->cmpw(Address::Absolute(Thread::ThreadFlagsOffset<kX86_64WordSize>().Int32Value(), + /* no_rip */ true), + Immediate(0)); if (successor == nullptr) { __ j(kNotEqual, slow_path->GetEntryLabel()); __ Bind(slow_path->GetReturnLabel()); @@ -5174,7 +5303,7 @@ void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck( Immediate(mirror::Class::kStatusInitialized)); __ j(kLess, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); - // No need for memory fence, thanks to the X86_64 memory model. + // No need for memory fence, thanks to the x86-64 memory model. } void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) { @@ -5205,32 +5334,16 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) { if (cls->IsReferrersClass()) { DCHECK(!cls->CanCallRuntime()); DCHECK(!cls->MustGenerateClinitCheck()); - uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value(); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_) - __ leaq(out, Address(current_method, declaring_class_offset)); - // /* mirror::Class* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc); - } else { - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - __ movl(out, Address(current_method, declaring_class_offset)); - } + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); } else { // /* GcRoot<mirror::Class>[] */ out = // current_method.ptr_sized_fields_->dex_cache_resolved_types_ __ movq(out, Address(current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86_64PointerSize).Int32Value())); - - size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex()); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::Class>* */ out = &out[type_index] - __ leaq(out, Address(out, cache_offset)); - // /* mirror::Class* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc); - } else { - // /* GcRoot<mirror::Class> */ out = out[type_index] - __ movl(out, Address(out, cache_offset)); - } + // /* GcRoot<mirror::Class> */ out = out[type_index] + GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); @@ -5283,30 +5396,14 @@ void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) { CpuRegister out = out_loc.AsRegister<CpuRegister>(); CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>(); - uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value(); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_) - __ leaq(out, Address(current_method, declaring_class_offset)); - // /* mirror::Class* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc); - } else { - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - __ movl(out, Address(current_method, declaring_class_offset)); - } - + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); - - size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex()); - if (kEmitCompilerReadBarrier) { - // /* GcRoot<mirror::String>* */ out = &out[string_index] - __ leaq(out, Address(out, cache_offset)); - // /* mirror::String* */ out = out->Read() - codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc); - } else { - // /* GcRoot<mirror::String> */ out = out[string_index] - __ movl(out, Address(out, cache_offset)); - } + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load); @@ -5318,7 +5415,8 @@ void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) { } static Address GetExceptionTlsAddress() { - return Address::Absolute(Thread::ExceptionOffset<kX86_64WordSize>().Int32Value(), true); + return Address::Absolute(Thread::ExceptionOffset<kX86_64WordSize>().Int32Value(), + /* no_rip */ true); } void LocationsBuilderX86_64::VisitLoadException(HLoadException* load) { @@ -5354,6 +5452,14 @@ void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) { CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); } +static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { + return kEmitCompilerReadBarrier && + (kUseBakerReadBarrier || + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck); +} + void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); @@ -5379,21 +5485,22 @@ void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) { locations->SetOut(Location::RequiresRegister()); // When read barriers are enabled, we need a temporary register for // some cases. - if (kEmitCompilerReadBarrier && - (type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + if (TypeCheckNeedsATemporary(type_check_kind)) { locations->AddTemp(Location::RequiresRegister()); } } void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); CpuRegister obj = obj_loc.AsRegister<CpuRegister>(); Location cls = locations->InAt(1); Location out_loc = locations->Out(); CpuRegister out = out_loc.AsRegister<CpuRegister>(); + Location temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? + locations->GetTemp(0) : + Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -5409,10 +5516,9 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { } // /* HeapReference<Class> */ out = obj->klass_ - __ movl(out, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, temp_loc); - switch (instruction->GetTypeCheckKind()) { + switch (type_check_kind) { case TypeCheckKind::kExactCheck: { if (cls.IsRegister()) { __ cmpl(out, cls.AsRegister<CpuRegister>()); @@ -5438,17 +5544,8 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { // object to avoid doing a comparison we know will fail. NearLabel loop, success; __ Bind(&loop); - Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `out` into `temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - CpuRegister temp = temp_loc.AsRegister<CpuRegister>(); - __ movl(temp, out); - } // /* HeapReference<Class> */ out = out->super_class_ - __ movl(out, Address(out, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, temp_loc); __ testl(out, out); // If `out` is null, we use it for the result, and jump to `done`. __ j(kEqual, &done); @@ -5477,17 +5574,8 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { __ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex())); } __ j(kEqual, &success); - Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `out` into `temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - CpuRegister temp = temp_loc.AsRegister<CpuRegister>(); - __ movl(temp, out); - } // /* HeapReference<Class> */ out = out->super_class_ - __ movl(out, Address(out, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, temp_loc); __ testl(out, out); __ j(kNotEqual, &loop); // If `out` is null, we use it for the result, and jump to `done`. @@ -5511,17 +5599,8 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { } __ j(kEqual, &exact_check); // Otherwise, we need to check that the object's class is a non-primitive array. - Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `out` into `temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - CpuRegister temp = temp_loc.AsRegister<CpuRegister>(); - __ movl(temp, out); - } // /* HeapReference<Class> */ out = out->component_type_ - __ movl(out, Address(out, component_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset); + GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, temp_loc); __ testl(out, out); // If `out` is null, we use it for the result, and jump to `done`. __ j(kEqual, &done); @@ -5565,6 +5644,13 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { // HInstanceOf instruction (following the runtime calling // convention), which might be cluttered by the potential first // read barrier emission at the beginning of this method. + // + // TODO: Introduce a new runtime entry point taking the object + // to test (instead of its class) as argument, and let it deal + // with the read barrier issues. This will let us refactor this + // case of the `switch` code as it was previously (with a direct + // call to the runtime not using a type checking slow path). + // This should also be beneficial for the other cases above. DCHECK(locations->OnlyCallsOnSlowPath()); slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(instruction, /* is_fatal */ false); @@ -5617,27 +5703,27 @@ void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) { locations->AddTemp(Location::RequiresRegister()); // When read barriers are enabled, we need an additional temporary // register for some cases. - if (kEmitCompilerReadBarrier && - (type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + if (TypeCheckNeedsATemporary(type_check_kind)) { locations->AddTemp(Location::RequiresRegister()); } } void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); CpuRegister obj = obj_loc.AsRegister<CpuRegister>(); Location cls = locations->InAt(1); Location temp_loc = locations->GetTemp(0); CpuRegister temp = temp_loc.AsRegister<CpuRegister>(); + Location temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? + locations->GetTemp(1) : + Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); bool is_type_check_slow_path_fatal = (type_check_kind == TypeCheckKind::kExactCheck || type_check_kind == TypeCheckKind::kAbstractClassCheck || @@ -5649,7 +5735,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { is_type_check_slow_path_fatal); codegen_->AddSlowPath(type_check_slow_path); - NearLabel done; + Label done; // Avoid null check if we know obj is not null. if (instruction->MustDoNullCheck()) { __ testl(obj, obj); @@ -5657,8 +5743,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { } // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); switch (type_check_kind) { case TypeCheckKind::kExactCheck: @@ -5680,18 +5765,8 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // object to avoid doing a comparison we know will fail. NearLabel loop, compare_classes; __ Bind(&loop); - Location temp2_loc = - kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `temp` into `temp2` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - CpuRegister temp2 = temp2_loc.AsRegister<CpuRegister>(); - __ movl(temp2, temp); - } // /* HeapReference<Class> */ temp = temp->super_class_ - __ movl(temp, Address(temp, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, temp2_loc); // If the class reference currently in `temp` is not null, jump // to the `compare_classes` label to compare it with the checked @@ -5704,8 +5779,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&compare_classes); @@ -5731,18 +5805,8 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { } __ j(kEqual, &done); - Location temp2_loc = - kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `temp` into `temp2` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - CpuRegister temp2 = temp2_loc.AsRegister<CpuRegister>(); - __ movl(temp2, temp); - } // /* HeapReference<Class> */ temp = temp->super_class_ - __ movl(temp, Address(temp, super_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset); + GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, temp2_loc); // If the class reference currently in `temp` is not null, jump // back at the beginning of the loop. @@ -5754,8 +5818,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); break; } @@ -5772,19 +5835,8 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { __ j(kEqual, &done); // Otherwise, we need to check that the object's class is a non-primitive array. - Location temp2_loc = - kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation(); - if (kEmitCompilerReadBarrier) { - // Save the value of `temp` into `temp2` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - CpuRegister temp2 = temp2_loc.AsRegister<CpuRegister>(); - __ movl(temp2, temp); - } // /* HeapReference<Class> */ temp = temp->component_type_ - __ movl(temp, Address(temp, component_offset)); - codegen_->MaybeGenerateReadBarrier( - instruction, temp_loc, temp_loc, temp2_loc, component_offset); + GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, temp2_loc); // If the component type is not null (i.e. the object is indeed // an array), jump to label `check_non_primitive_component_type` @@ -5798,8 +5850,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // going into the slow path, as it has been overwritten in the // meantime. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); __ Bind(&check_non_primitive_component_type); @@ -5807,8 +5858,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { __ j(kEqual, &done); // Same comment as above regarding `temp` and the slow path. // /* HeapReference<Class> */ temp = obj->klass_ - __ movl(temp, Address(obj, class_offset)); - codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc); __ jmp(type_check_slow_path->GetEntryLabel()); break; } @@ -5825,6 +5875,13 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // instruction (following the runtime calling convention), which // might be cluttered by the potential first read barrier // emission at the beginning of this method. + // + // TODO: Introduce a new runtime entry point taking the object + // to test (instead of its class) as argument, and let it deal + // with the read barrier issues. This will let us refactor this + // case of the `switch` code as it was previously (with a direct + // call to the runtime not using a type checking slow path). + // This should also be beneficial for the other cases above. __ jmp(type_check_slow_path->GetEntryLabel()); break; } @@ -5968,14 +6025,227 @@ void InstructionCodeGeneratorX86_64::HandleBitwiseOperation(HBinaryOperation* in } } -void CodeGeneratorX86_64::GenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { +void InstructionCodeGeneratorX86_64::GenerateReferenceLoadOneRegister(HInstruction* instruction, + Location out, + uint32_t offset, + Location temp) { + CpuRegister out_reg = out.AsRegister<CpuRegister>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, out_reg, offset, temp, /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // Save the value of `out` into `temp` before overwriting it + // in the following move operation, as we will need it for the + // read barrier below. + __ movl(temp.AsRegister<CpuRegister>(), out_reg); + // /* HeapReference<Object> */ out = *(out + offset) + __ movl(out_reg, Address(out_reg, offset)); + codegen_->GenerateReadBarrierSlow(instruction, out, out, temp, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + __ movl(out_reg, Address(out_reg, offset)); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + +void InstructionCodeGeneratorX86_64::GenerateReferenceLoadTwoRegisters(HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location temp) { + CpuRegister out_reg = out.AsRegister<CpuRegister>(); + CpuRegister obj_reg = obj.AsRegister<CpuRegister>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier( + instruction, out, obj_reg, offset, temp, /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ movl(out_reg, Address(obj_reg, offset)); + codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ movl(out_reg, Address(obj_reg, offset)); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + +void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + CpuRegister obj, + uint32_t offset) { + CpuRegister root_reg = root.AsRegister<CpuRegister>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + // Fast path implementation of art::ReadBarrier::BarrierForRoot when + // Baker's read barrier are used: + // + // root = obj.field; + // if (Thread::Current()->GetIsGcMarking()) { + // root = ReadBarrier::Mark(root) + // } + + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + __ movl(root_reg, Address(obj, offset)); + static_assert( + sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), + "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " + "have different sizes."); + static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::CompressedReference<mirror::Object> and int32_t " + "have different sizes."); + + // Slow path used to mark the GC root `root`. + SlowPathCode* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86_64(instruction, root, root); + codegen_->AddSlowPath(slow_path); + + __ gs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86_64WordSize>().Int32Value(), + /* no_rip */ true), + Immediate(0)); + __ j(kNotEqual, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); + } else { + // GC root loaded through a slow path for read barriers other + // than Baker's. + // /* GcRoot<mirror::Object>* */ root = obj + offset + __ leaq(root_reg, Address(obj, offset)); + // /* mirror::Object* */ root = root->Read() + codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); + } + } else { + // Plain GC root load with no read barrier. + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + __ movl(root_reg, Address(obj, offset)); + } +} + +void CodeGeneratorX86_64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + CpuRegister obj, + uint32_t offset, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // /* HeapReference<Object> */ ref = *(obj + offset) + Address src(obj, offset); + GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check); +} + +void CodeGeneratorX86_64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + CpuRegister obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // /* HeapReference<Object> */ ref = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + Address src = index.IsConstant() ? + Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset) : + Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset); + GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check); +} + +void CodeGeneratorX86_64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + CpuRegister obj, + const Address& src, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // In slow path based read barriers, the read barrier call is + // inserted after the original load. However, in fast path based + // Baker's read barriers, we need to perform the load of + // mirror::Object::monitor_ *before* the original reference load. + // This load-load ordering is required by the read barrier. + // The fast path/slow path (for Baker's algorithm) should look like: + // + // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); + // lfence; // Load fence or artificial data dependency to prevent load-load reordering + // HeapReference<Object> ref = *src; // Original reference load. + // bool is_gray = (rb_state == ReadBarrier::gray_ptr_); + // if (is_gray) { + // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. + // } + // + // Note: the original implementation in ReadBarrier::Barrier is + // slightly more complex as: + // - it implements the load-load fence using a data dependency on + // the high-bits of rb_state, which are expected to be all zeroes; + // - it performs additional checks that we do not do here for + // performance reasons. + + CpuRegister ref_reg = ref.AsRegister<CpuRegister>(); + CpuRegister temp_reg = temp.AsRegister<CpuRegister>(); + uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); + + // /* int32_t */ monitor = obj->monitor_ + __ movl(temp_reg, Address(obj, monitor_offset)); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + // /* LockWord */ lock_word = LockWord(monitor) + static_assert(sizeof(LockWord) == sizeof(int32_t), + "art::LockWord and int32_t have different sizes."); + // /* uint32_t */ rb_state = lock_word.ReadBarrierState() + __ shrl(temp_reg, Immediate(LockWord::kReadBarrierStateShift)); + __ andl(temp_reg, Immediate(LockWord::kReadBarrierStateMask)); + static_assert( + LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_, + "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_."); + + // Load fence to prevent load-load reordering. + // Note that this is a no-op, thanks to the x86-64 memory model. + GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + + // The actual reference load. + // /* HeapReference<Object> */ ref = *src + __ movl(ref_reg, src); + + // Object* ref = ref_addr->AsMirrorPtr() + __ MaybeUnpoisonHeapReference(ref_reg); + + // Slow path used to mark the object `ref` when it is gray. + SlowPathCode* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86_64(instruction, ref, ref); + AddSlowPath(slow_path); + + // if (rb_state == ReadBarrier::gray_ptr_) + // ref = ReadBarrier::Mark(ref); + __ cmpl(temp_reg, Immediate(ReadBarrier::gray_ptr_)); + __ j(kEqual, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + +void CodeGeneratorX86_64::GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { DCHECK(kEmitCompilerReadBarrier); + // Insert a slow path based read barrier *after* the reference load. + // // If heap poisoning is enabled, the unpoisoning of the loaded // reference will be carried out by the runtime within the slow // path. @@ -5989,57 +6259,41 @@ void CodeGeneratorX86_64::GenerateReadBarrier(HInstruction* instruction, ReadBarrierForHeapReferenceSlowPathX86_64(instruction, out, ref, obj, offset, index); AddSlowPath(slow_path); - // TODO: When read barrier has a fast path, add it here. - /* Currently the read barrier call is inserted after the original load. - * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the - * original load. This load-load ordering is required by the read barrier. - * The fast path/slow path (for Baker's algorithm) should look like: - * - * bool isGray = obj.LockWord & kReadBarrierMask; - * lfence; // load fence or artificial data dependence to prevent load-load reordering - * ref = obj.field; // this is the original load - * if (isGray) { - * ref = Mark(ref); // ideally the slow path just does Mark(ref) - * } - */ - __ jmp(slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } -void CodeGeneratorX86_64::MaybeGenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { +void CodeGeneratorX86_64::MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { if (kEmitCompilerReadBarrier) { + // Baker's read barriers shall be handled by the fast path + // (CodeGeneratorX86_64::GenerateReferenceLoadWithBakerReadBarrier). + DCHECK(!kUseBakerReadBarrier); // If heap poisoning is enabled, unpoisoning will be taken care of // by the runtime within the slow path. - GenerateReadBarrier(instruction, out, ref, obj, offset, index); + GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); } else if (kPoisonHeapReferences) { __ UnpoisonHeapReference(out.AsRegister<CpuRegister>()); } } -void CodeGeneratorX86_64::GenerateReadBarrierForRoot(HInstruction* instruction, - Location out, - Location root) { +void CodeGeneratorX86_64::GenerateReadBarrierForRootSlow(HInstruction* instruction, + Location out, + Location root) { DCHECK(kEmitCompilerReadBarrier); + // Insert a slow path based read barrier *after* the GC root load. + // // Note that GC roots are not affected by heap poisoning, so we do // not need to do anything special for this here. SlowPathCode* slow_path = new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86_64(instruction, out, root); AddSlowPath(slow_path); - // TODO: Implement a fast path for ReadBarrierForRoot, performing - // the following operation (for Baker's algorithm): - // - // if (thread.tls32_.is_gc_marking) { - // root = Mark(root); - // } - __ jmp(slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } @@ -6077,58 +6331,11 @@ void LocationsBuilderX86_64::VisitPackedSwitch(HPackedSwitch* switch_instr) { void InstructionCodeGeneratorX86_64::VisitPackedSwitch(HPackedSwitch* switch_instr) { int32_t lower_bound = switch_instr->GetStartValue(); - uint32_t num_entries = switch_instr->GetNumEntries(); + int32_t num_entries = switch_instr->GetNumEntries(); LocationSummary* locations = switch_instr->GetLocations(); CpuRegister value_reg_in = locations->InAt(0).AsRegister<CpuRegister>(); CpuRegister temp_reg = locations->GetTemp(0).AsRegister<CpuRegister>(); CpuRegister base_reg = locations->GetTemp(1).AsRegister<CpuRegister>(); - HBasicBlock* default_block = switch_instr->GetDefaultBlock(); - - // Should we generate smaller inline compare/jumps? - if (num_entries <= kPackedSwitchJumpTableThreshold) { - // Figure out the correct compare values and jump conditions. - // Handle the first compare/branch as a special case because it might - // jump to the default case. - DCHECK_GT(num_entries, 2u); - Condition first_condition; - uint32_t index; - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - if (lower_bound != 0) { - first_condition = kLess; - __ cmpl(value_reg_in, Immediate(lower_bound)); - __ j(first_condition, codegen_->GetLabelOf(default_block)); - __ j(kEqual, codegen_->GetLabelOf(successors[0])); - - index = 1; - } else { - // Handle all the compare/jumps below. - first_condition = kBelow; - index = 0; - } - - // Handle the rest of the compare/jumps. - for (; index + 1 < num_entries; index += 2) { - int32_t compare_to_value = lower_bound + index + 1; - __ cmpl(value_reg_in, Immediate(compare_to_value)); - // Jump to successors[index] if value < case_value[index]. - __ j(first_condition, codegen_->GetLabelOf(successors[index])); - // Jump to successors[index + 1] if value == case_value[index + 1]. - __ j(kEqual, codegen_->GetLabelOf(successors[index + 1])); - } - - if (index != num_entries) { - // There are an odd number of entries. Handle the last one. - DCHECK_EQ(index + 1, num_entries); - __ cmpl(value_reg_in, Immediate(lower_bound + index)); - __ j(kEqual, codegen_->GetLabelOf(successors[index])); - } - - // And the default for any other value. - if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { - __ jmp(codegen_->GetLabelOf(default_block)); - } - return; - } // Remove the bias, if needed. Register value_reg_out = value_reg_in.AsRegister(); @@ -6139,6 +6346,7 @@ void InstructionCodeGeneratorX86_64::VisitPackedSwitch(HPackedSwitch* switch_ins CpuRegister value_reg(value_reg_out); // Is the value in range? + HBasicBlock* default_block = switch_instr->GetDefaultBlock(); __ cmpl(value_reg, Immediate(num_entries - 1)); __ j(kAbove, codegen_->GetLabelOf(default_block)); @@ -6288,7 +6496,7 @@ Address CodeGeneratorX86_64::LiteralInt64Address(int64_t v) { // TODO: trg as memory. void CodeGeneratorX86_64::MoveFromReturnRegister(Location trg, Primitive::Type type) { if (!trg.IsValid()) { - DCHECK(type == Primitive::kPrimVoid); + DCHECK_EQ(type, Primitive::kPrimVoid); return; } diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 7351fed0fa..dda9ea22d9 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -213,11 +213,44 @@ class InstructionCodeGeneratorX86_64 : public HGraphVisitor { void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction); void GenerateDivRemIntegral(HBinaryOperation* instruction); void HandleShift(HBinaryOperation* operation); - void GenerateMemoryBarrier(MemBarrierKind kind); + void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info, bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); + + // Generate a heap reference load using one register `out`: + // + // out <- *(out + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // Register `temp` is used when generating a read barrier. + void GenerateReferenceLoadOneRegister(HInstruction* instruction, + Location out, + uint32_t offset, + Location temp); + // Generate a heap reference load using two different registers + // `out` and `obj`: + // + // out <- *(obj + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // Register `temp` is used when generating a Baker's read barrier. + void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location temp); + // Generate a GC root reference load: + // + // root <- *(obj + offset) + // + // while honoring read barriers (if any). + void GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + CpuRegister obj, + uint32_t offset); + void GenerateImplicitNullCheck(HNullCheck* instruction); void GenerateExplicitNullCheck(HNullCheck* instruction); void PushOntoFPStack(Location source, uint32_t temp_offset, @@ -324,6 +357,8 @@ class CodeGeneratorX86_64 : public CodeGenerator { CpuRegister value, bool value_can_be_null); + void GenerateMemoryBarrier(MemBarrierKind kind); + // Helper method to move a value between two locations. void Move(Location destination, Location source); @@ -356,7 +391,26 @@ class CodeGeneratorX86_64 : public CodeGenerator { return isa_features_; } - // Generate a read barrier for a heap reference within `instruction`. + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference field load when Baker's read barriers are used. + void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location out, + CpuRegister obj, + uint32_t offset, + Location temp, + bool needs_null_check); + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference array load when Baker's read barriers are used. + void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location out, + CpuRegister obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check); + + // Generate a read barrier for a heap reference within `instruction` + // using a slow path. // // A read barrier for an object reference read from the heap is // implemented as a call to the artReadBarrierSlow runtime entry @@ -373,23 +427,25 @@ class CodeGeneratorX86_64 : public CodeGenerator { // When `index` provided (i.e., when it is different from // Location::NoLocation()), the offset value passed to // artReadBarrierSlow is adjusted to take `index` into account. - void GenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // If read barriers are enabled, generate a read barrier for a heap reference. - // If heap poisoning is enabled, also unpoison the reference in `out`. - void MaybeGenerateReadBarrier(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // Generate a read barrier for a GC root within `instruction`. + void GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // If read barriers are enabled, generate a read barrier for a heap + // reference using a slow path. If heap poisoning is enabled, also + // unpoison the reference in `out`. + void MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // Generate a read barrier for a GC root within `instruction` using + // a slow path. // // A read barrier for an object reference GC root is implemented as // a call to the artReadBarrierForRootSlow runtime entry point, @@ -399,7 +455,7 @@ class CodeGeneratorX86_64 : public CodeGenerator { // // The `out` location contains the value returned by // artReadBarrierForRootSlow. - void GenerateReadBarrierForRoot(HInstruction* instruction, Location out, Location root); + void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); int ConstantAreaStart() const { return constant_area_start_; @@ -424,6 +480,15 @@ class CodeGeneratorX86_64 : public CodeGenerator { HInstruction* instruction); private: + // Factored implementation of GenerateFieldLoadWithBakerReadBarrier + // and GenerateArrayLoadWithBakerReadBarrier. + void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + CpuRegister obj, + const Address& src, + Location temp, + bool needs_null_check); + struct PcRelativeDexCacheAccessInfo { PcRelativeDexCacheAccessInfo(const DexFile& dex_file, uint32_t element_off) : target_dex_file(dex_file), element_offset(element_off), label() { } diff --git a/compiler/optimizing/codegen_test.cc b/compiler/optimizing/codegen_test.cc index 57de41f557..d970704368 100644 --- a/compiler/optimizing/codegen_test.cc +++ b/compiler/optimizing/codegen_test.cc @@ -35,6 +35,7 @@ #include "code_generator_mips64.h" #include "code_generator_x86.h" #include "code_generator_x86_64.h" +#include "code_simulator_container.h" #include "common_compiler_test.h" #include "dex_file.h" #include "dex_instruction.h" @@ -124,26 +125,85 @@ class InternalCodeAllocator : public CodeAllocator { DISALLOW_COPY_AND_ASSIGN(InternalCodeAllocator); }; +static bool CanExecuteOnHardware(InstructionSet target_isa) { + return (target_isa == kRuntimeISA) + // Handle the special case of ARM, with two instructions sets (ARM32 and Thumb-2). + || (kRuntimeISA == kArm && target_isa == kThumb2); +} + +static bool CanExecute(InstructionSet target_isa) { + CodeSimulatorContainer simulator(target_isa); + return CanExecuteOnHardware(target_isa) || simulator.CanSimulate(); +} + +template <typename Expected> +static Expected SimulatorExecute(CodeSimulator* simulator, Expected (*f)()); + +template <> +bool SimulatorExecute<bool>(CodeSimulator* simulator, bool (*f)()) { + simulator->RunFrom(reinterpret_cast<intptr_t>(f)); + return simulator->GetCReturnBool(); +} + +template <> +int32_t SimulatorExecute<int32_t>(CodeSimulator* simulator, int32_t (*f)()) { + simulator->RunFrom(reinterpret_cast<intptr_t>(f)); + return simulator->GetCReturnInt32(); +} + +template <> +int64_t SimulatorExecute<int64_t>(CodeSimulator* simulator, int64_t (*f)()) { + simulator->RunFrom(reinterpret_cast<intptr_t>(f)); + return simulator->GetCReturnInt64(); +} + +template <typename Expected> +static void VerifyGeneratedCode(InstructionSet target_isa, + Expected (*f)(), + bool has_result, + Expected expected) { + ASSERT_TRUE(CanExecute(target_isa)) << "Target isa is not executable."; + + // Verify on simulator. + CodeSimulatorContainer simulator(target_isa); + if (simulator.CanSimulate()) { + Expected result = SimulatorExecute<Expected>(simulator.Get(), f); + if (has_result) { + ASSERT_EQ(expected, result); + } + } + + // Verify on hardware. + if (CanExecuteOnHardware(target_isa)) { + Expected result = f(); + if (has_result) { + ASSERT_EQ(expected, result); + } + } +} + template <typename Expected> static void Run(const InternalCodeAllocator& allocator, const CodeGenerator& codegen, bool has_result, Expected expected) { + InstructionSet target_isa = codegen.GetInstructionSet(); + typedef Expected (*fptr)(); CommonCompilerTest::MakeExecutable(allocator.GetMemory(), allocator.GetSize()); fptr f = reinterpret_cast<fptr>(allocator.GetMemory()); - if (codegen.GetInstructionSet() == kThumb2) { + if (target_isa == kThumb2) { // For thumb we need the bottom bit set. f = reinterpret_cast<fptr>(reinterpret_cast<uintptr_t>(f) + 1); } - Expected result = f(); - if (has_result) { - ASSERT_EQ(expected, result); - } + VerifyGeneratedCode(target_isa, f, has_result, expected); } template <typename Expected> -static void RunCodeBaseline(HGraph* graph, bool has_result, Expected expected) { +static void RunCodeBaseline(InstructionSet target_isa, + HGraph* graph, + bool has_result, + Expected expected) { InternalCodeAllocator allocator; CompilerOptions compiler_options; @@ -153,7 +213,7 @@ static void RunCodeBaseline(HGraph* graph, bool has_result, Expected expected) { // We avoid doing a stack overflow check that requires the runtime being setup, // by making sure the compiler knows the methods we are running are leaf methods. codegenX86.CompileBaseline(&allocator, true); - if (kRuntimeISA == kX86) { + if (target_isa == kX86) { Run(allocator, codegenX86, has_result, expected); } @@ -161,7 +221,7 @@ static void RunCodeBaseline(HGraph* graph, bool has_result, Expected expected) { ArmInstructionSetFeatures::FromCppDefines()); TestCodeGeneratorARM codegenARM(graph, *features_arm.get(), compiler_options); codegenARM.CompileBaseline(&allocator, true); - if (kRuntimeISA == kArm || kRuntimeISA == kThumb2) { + if (target_isa == kArm || target_isa == kThumb2) { Run(allocator, codegenARM, has_result, expected); } @@ -169,7 +229,7 @@ static void RunCodeBaseline(HGraph* graph, bool has_result, Expected expected) { X86_64InstructionSetFeatures::FromCppDefines()); x86_64::CodeGeneratorX86_64 codegenX86_64(graph, *features_x86_64.get(), compiler_options); codegenX86_64.CompileBaseline(&allocator, true); - if (kRuntimeISA == kX86_64) { + if (target_isa == kX86_64) { Run(allocator, codegenX86_64, has_result, expected); } @@ -177,7 +237,7 @@ static void RunCodeBaseline(HGraph* graph, bool has_result, Expected expected) { Arm64InstructionSetFeatures::FromCppDefines()); arm64::CodeGeneratorARM64 codegenARM64(graph, *features_arm64.get(), compiler_options); codegenARM64.CompileBaseline(&allocator, true); - if (kRuntimeISA == kArm64) { + if (target_isa == kArm64) { Run(allocator, codegenARM64, has_result, expected); } @@ -193,7 +253,7 @@ static void RunCodeBaseline(HGraph* graph, bool has_result, Expected expected) { Mips64InstructionSetFeatures::FromCppDefines()); mips64::CodeGeneratorMIPS64 codegenMIPS64(graph, *features_mips64.get(), compiler_options); codegenMIPS64.CompileBaseline(&allocator, true); - if (kRuntimeISA == kMips64) { + if (target_isa == kMips64) { Run(allocator, codegenMIPS64, has_result, expected); } } @@ -221,37 +281,38 @@ static void RunCodeOptimized(CodeGenerator* codegen, } template <typename Expected> -static void RunCodeOptimized(HGraph* graph, +static void RunCodeOptimized(InstructionSet target_isa, + HGraph* graph, std::function<void(HGraph*)> hook_before_codegen, bool has_result, Expected expected) { CompilerOptions compiler_options; - if (kRuntimeISA == kArm || kRuntimeISA == kThumb2) { - TestCodeGeneratorARM codegenARM(graph, - *ArmInstructionSetFeatures::FromCppDefines(), - compiler_options); + if (target_isa == kArm || target_isa == kThumb2) { + std::unique_ptr<const ArmInstructionSetFeatures> features_arm( + ArmInstructionSetFeatures::FromCppDefines()); + TestCodeGeneratorARM codegenARM(graph, *features_arm.get(), compiler_options); RunCodeOptimized(&codegenARM, graph, hook_before_codegen, has_result, expected); - } else if (kRuntimeISA == kArm64) { - arm64::CodeGeneratorARM64 codegenARM64(graph, - *Arm64InstructionSetFeatures::FromCppDefines(), - compiler_options); + } else if (target_isa == kArm64) { + std::unique_ptr<const Arm64InstructionSetFeatures> features_arm64( + Arm64InstructionSetFeatures::FromCppDefines()); + arm64::CodeGeneratorARM64 codegenARM64(graph, *features_arm64.get(), compiler_options); RunCodeOptimized(&codegenARM64, graph, hook_before_codegen, has_result, expected); - } else if (kRuntimeISA == kX86) { + } else if (target_isa == kX86) { std::unique_ptr<const X86InstructionSetFeatures> features_x86( X86InstructionSetFeatures::FromCppDefines()); x86::CodeGeneratorX86 codegenX86(graph, *features_x86.get(), compiler_options); RunCodeOptimized(&codegenX86, graph, hook_before_codegen, has_result, expected); - } else if (kRuntimeISA == kX86_64) { + } else if (target_isa == kX86_64) { std::unique_ptr<const X86_64InstructionSetFeatures> features_x86_64( X86_64InstructionSetFeatures::FromCppDefines()); x86_64::CodeGeneratorX86_64 codegenX86_64(graph, *features_x86_64.get(), compiler_options); RunCodeOptimized(&codegenX86_64, graph, hook_before_codegen, has_result, expected); - } else if (kRuntimeISA == kMips) { + } else if (target_isa == kMips) { std::unique_ptr<const MipsInstructionSetFeatures> features_mips( MipsInstructionSetFeatures::FromCppDefines()); mips::CodeGeneratorMIPS codegenMIPS(graph, *features_mips.get(), compiler_options); RunCodeOptimized(&codegenMIPS, graph, hook_before_codegen, has_result, expected); - } else if (kRuntimeISA == kMips64) { + } else if (target_isa == kMips64) { std::unique_ptr<const Mips64InstructionSetFeatures> features_mips64( Mips64InstructionSetFeatures::FromCppDefines()); mips64::CodeGeneratorMIPS64 codegenMIPS64(graph, *features_mips64.get(), compiler_options); @@ -259,7 +320,10 @@ static void RunCodeOptimized(HGraph* graph, } } -static void TestCode(const uint16_t* data, bool has_result = false, int32_t expected = 0) { +static void TestCode(InstructionSet target_isa, + const uint16_t* data, + bool has_result = false, + int32_t expected = 0) { ArenaPool pool; ArenaAllocator arena(&pool); HGraph* graph = CreateGraph(&arena); @@ -269,10 +333,13 @@ static void TestCode(const uint16_t* data, bool has_result = false, int32_t expe ASSERT_TRUE(graph_built); // Remove suspend checks, they cannot be executed in this context. RemoveSuspendChecks(graph); - RunCodeBaseline(graph, has_result, expected); + RunCodeBaseline(target_isa, graph, has_result, expected); } -static void TestCodeLong(const uint16_t* data, bool has_result, int64_t expected) { +static void TestCodeLong(InstructionSet target_isa, + const uint16_t* data, + bool has_result, + int64_t expected) { ArenaPool pool; ArenaAllocator arena(&pool); HGraph* graph = CreateGraph(&arena); @@ -282,108 +349,110 @@ static void TestCodeLong(const uint16_t* data, bool has_result, int64_t expected ASSERT_TRUE(graph_built); // Remove suspend checks, they cannot be executed in this context. RemoveSuspendChecks(graph); - RunCodeBaseline(graph, has_result, expected); + RunCodeBaseline(target_isa, graph, has_result, expected); } -TEST(CodegenTest, ReturnVoid) { +class CodegenTest: public ::testing::TestWithParam<InstructionSet> {}; + +TEST_P(CodegenTest, ReturnVoid) { const uint16_t data[] = ZERO_REGISTER_CODE_ITEM(Instruction::RETURN_VOID); - TestCode(data); + TestCode(GetParam(), data); } -TEST(CodegenTest, CFG1) { +TEST_P(CodegenTest, CFG1) { const uint16_t data[] = ZERO_REGISTER_CODE_ITEM( Instruction::GOTO | 0x100, Instruction::RETURN_VOID); - TestCode(data); + TestCode(GetParam(), data); } -TEST(CodegenTest, CFG2) { +TEST_P(CodegenTest, CFG2) { const uint16_t data[] = ZERO_REGISTER_CODE_ITEM( Instruction::GOTO | 0x100, Instruction::GOTO | 0x100, Instruction::RETURN_VOID); - TestCode(data); + TestCode(GetParam(), data); } -TEST(CodegenTest, CFG3) { +TEST_P(CodegenTest, CFG3) { const uint16_t data1[] = ZERO_REGISTER_CODE_ITEM( Instruction::GOTO | 0x200, Instruction::RETURN_VOID, Instruction::GOTO | 0xFF00); - TestCode(data1); + TestCode(GetParam(), data1); const uint16_t data2[] = ZERO_REGISTER_CODE_ITEM( Instruction::GOTO_16, 3, Instruction::RETURN_VOID, Instruction::GOTO_16, 0xFFFF); - TestCode(data2); + TestCode(GetParam(), data2); const uint16_t data3[] = ZERO_REGISTER_CODE_ITEM( Instruction::GOTO_32, 4, 0, Instruction::RETURN_VOID, Instruction::GOTO_32, 0xFFFF, 0xFFFF); - TestCode(data3); + TestCode(GetParam(), data3); } -TEST(CodegenTest, CFG4) { +TEST_P(CodegenTest, CFG4) { const uint16_t data[] = ZERO_REGISTER_CODE_ITEM( Instruction::RETURN_VOID, Instruction::GOTO | 0x100, Instruction::GOTO | 0xFE00); - TestCode(data); + TestCode(GetParam(), data); } -TEST(CodegenTest, CFG5) { +TEST_P(CodegenTest, CFG5) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 0 | 0, Instruction::IF_EQ, 3, Instruction::GOTO | 0x100, Instruction::RETURN_VOID); - TestCode(data); + TestCode(GetParam(), data); } -TEST(CodegenTest, IntConstant) { +TEST_P(CodegenTest, IntConstant) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 0 | 0, Instruction::RETURN_VOID); - TestCode(data); + TestCode(GetParam(), data); } -TEST(CodegenTest, Return1) { +TEST_P(CodegenTest, Return1) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 0 | 0, Instruction::RETURN | 0); - TestCode(data, true, 0); + TestCode(GetParam(), data, true, 0); } -TEST(CodegenTest, Return2) { +TEST_P(CodegenTest, Return2) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 | 0, Instruction::CONST_4 | 0 | 1 << 8, Instruction::RETURN | 1 << 8); - TestCode(data, true, 0); + TestCode(GetParam(), data, true, 0); } -TEST(CodegenTest, Return3) { +TEST_P(CodegenTest, Return3) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 | 0, Instruction::CONST_4 | 1 << 8 | 1 << 12, Instruction::RETURN | 1 << 8); - TestCode(data, true, 1); + TestCode(GetParam(), data, true, 1); } -TEST(CodegenTest, ReturnIf1) { +TEST_P(CodegenTest, ReturnIf1) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 | 0, Instruction::CONST_4 | 1 << 8 | 1 << 12, @@ -391,10 +460,10 @@ TEST(CodegenTest, ReturnIf1) { Instruction::RETURN | 0 << 8, Instruction::RETURN | 1 << 8); - TestCode(data, true, 1); + TestCode(GetParam(), data, true, 1); } -TEST(CodegenTest, ReturnIf2) { +TEST_P(CodegenTest, ReturnIf2) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 | 0, Instruction::CONST_4 | 1 << 8 | 1 << 12, @@ -402,12 +471,12 @@ TEST(CodegenTest, ReturnIf2) { Instruction::RETURN | 0 << 8, Instruction::RETURN | 1 << 8); - TestCode(data, true, 0); + TestCode(GetParam(), data, true, 0); } // Exercise bit-wise (one's complement) not-int instruction. #define NOT_INT_TEST(TEST_NAME, INPUT, EXPECTED_OUTPUT) \ -TEST(CodegenTest, TEST_NAME) { \ +TEST_P(CodegenTest, TEST_NAME) { \ const int32_t input = INPUT; \ const uint16_t input_lo = Low16Bits(input); \ const uint16_t input_hi = High16Bits(input); \ @@ -416,7 +485,7 @@ TEST(CodegenTest, TEST_NAME) { \ Instruction::NOT_INT | 1 << 8 | 0 << 12 , \ Instruction::RETURN | 1 << 8); \ \ - TestCode(data, true, EXPECTED_OUTPUT); \ + TestCode(GetParam(), data, true, EXPECTED_OUTPUT); \ } NOT_INT_TEST(ReturnNotIntMinus2, -2, 1) @@ -432,7 +501,7 @@ NOT_INT_TEST(ReturnNotIntINT32_MAX, 2147483647, -2147483648) // -(2^31) // Exercise bit-wise (one's complement) not-long instruction. #define NOT_LONG_TEST(TEST_NAME, INPUT, EXPECTED_OUTPUT) \ -TEST(CodegenTest, TEST_NAME) { \ +TEST_P(CodegenTest, TEST_NAME) { \ const int64_t input = INPUT; \ const uint16_t word0 = Low16Bits(Low32Bits(input)); /* LSW. */ \ const uint16_t word1 = High16Bits(Low32Bits(input)); \ @@ -443,7 +512,7 @@ TEST(CodegenTest, TEST_NAME) { \ Instruction::NOT_LONG | 2 << 8 | 0 << 12, \ Instruction::RETURN_WIDE | 2 << 8); \ \ - TestCodeLong(data, true, EXPECTED_OUTPUT); \ + TestCodeLong(GetParam(), data, true, EXPECTED_OUTPUT); \ } NOT_LONG_TEST(ReturnNotLongMinus2, INT64_C(-2), INT64_C(1)) @@ -482,7 +551,7 @@ NOT_LONG_TEST(ReturnNotLongINT64_MAX, #undef NOT_LONG_TEST -TEST(CodegenTest, IntToLongOfLongToInt) { +TEST_P(CodegenTest, IntToLongOfLongToInt) { const int64_t input = INT64_C(4294967296); // 2^32 const uint16_t word0 = Low16Bits(Low32Bits(input)); // LSW. const uint16_t word1 = High16Bits(Low32Bits(input)); @@ -496,48 +565,48 @@ TEST(CodegenTest, IntToLongOfLongToInt) { Instruction::INT_TO_LONG | 2 << 8 | 4 << 12, Instruction::RETURN_WIDE | 2 << 8); - TestCodeLong(data, true, 1); + TestCodeLong(GetParam(), data, true, 1); } -TEST(CodegenTest, ReturnAdd1) { +TEST_P(CodegenTest, ReturnAdd1) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 3 << 12 | 0, Instruction::CONST_4 | 4 << 12 | 1 << 8, Instruction::ADD_INT, 1 << 8 | 0, Instruction::RETURN); - TestCode(data, true, 7); + TestCode(GetParam(), data, true, 7); } -TEST(CodegenTest, ReturnAdd2) { +TEST_P(CodegenTest, ReturnAdd2) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 3 << 12 | 0, Instruction::CONST_4 | 4 << 12 | 1 << 8, Instruction::ADD_INT_2ADDR | 1 << 12, Instruction::RETURN); - TestCode(data, true, 7); + TestCode(GetParam(), data, true, 7); } -TEST(CodegenTest, ReturnAdd3) { +TEST_P(CodegenTest, ReturnAdd3) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 4 << 12 | 0 << 8, Instruction::ADD_INT_LIT8, 3 << 8 | 0, Instruction::RETURN); - TestCode(data, true, 7); + TestCode(GetParam(), data, true, 7); } -TEST(CodegenTest, ReturnAdd4) { +TEST_P(CodegenTest, ReturnAdd4) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 4 << 12 | 0 << 8, Instruction::ADD_INT_LIT16, 3, Instruction::RETURN); - TestCode(data, true, 7); + TestCode(GetParam(), data, true, 7); } -TEST(CodegenTest, NonMaterializedCondition) { +TEST_P(CodegenTest, NonMaterializedCondition) { ArenaPool pool; ArenaAllocator allocator(&pool); @@ -583,30 +652,30 @@ TEST(CodegenTest, NonMaterializedCondition) { block->InsertInstructionBefore(move, block->GetLastInstruction()); }; - RunCodeOptimized(graph, hook_before_codegen, true, 0); + RunCodeOptimized(GetParam(), graph, hook_before_codegen, true, 0); } -TEST(CodegenTest, ReturnMulInt) { +TEST_P(CodegenTest, ReturnMulInt) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 3 << 12 | 0, Instruction::CONST_4 | 4 << 12 | 1 << 8, Instruction::MUL_INT, 1 << 8 | 0, Instruction::RETURN); - TestCode(data, true, 12); + TestCode(GetParam(), data, true, 12); } -TEST(CodegenTest, ReturnMulInt2addr) { +TEST_P(CodegenTest, ReturnMulInt2addr) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 3 << 12 | 0, Instruction::CONST_4 | 4 << 12 | 1 << 8, Instruction::MUL_INT_2ADDR | 1 << 12, Instruction::RETURN); - TestCode(data, true, 12); + TestCode(GetParam(), data, true, 12); } -TEST(CodegenTest, ReturnMulLong) { +TEST_P(CodegenTest, ReturnMulLong) { const uint16_t data[] = FOUR_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 3 << 12 | 0, Instruction::CONST_4 | 0 << 12 | 1 << 8, @@ -615,10 +684,10 @@ TEST(CodegenTest, ReturnMulLong) { Instruction::MUL_LONG, 2 << 8 | 0, Instruction::RETURN_WIDE); - TestCodeLong(data, true, 12); + TestCodeLong(GetParam(), data, true, 12); } -TEST(CodegenTest, ReturnMulLong2addr) { +TEST_P(CodegenTest, ReturnMulLong2addr) { const uint16_t data[] = FOUR_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 3 << 12 | 0 << 8, Instruction::CONST_4 | 0 << 12 | 1 << 8, @@ -627,28 +696,28 @@ TEST(CodegenTest, ReturnMulLong2addr) { Instruction::MUL_LONG_2ADDR | 2 << 12, Instruction::RETURN_WIDE); - TestCodeLong(data, true, 12); + TestCodeLong(GetParam(), data, true, 12); } -TEST(CodegenTest, ReturnMulIntLit8) { +TEST_P(CodegenTest, ReturnMulIntLit8) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 4 << 12 | 0 << 8, Instruction::MUL_INT_LIT8, 3 << 8 | 0, Instruction::RETURN); - TestCode(data, true, 12); + TestCode(GetParam(), data, true, 12); } -TEST(CodegenTest, ReturnMulIntLit16) { +TEST_P(CodegenTest, ReturnMulIntLit16) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 4 << 12 | 0 << 8, Instruction::MUL_INT_LIT16, 3, Instruction::RETURN); - TestCode(data, true, 12); + TestCode(GetParam(), data, true, 12); } -TEST(CodegenTest, MaterializedCondition1) { +TEST_P(CodegenTest, MaterializedCondition1) { // Check that condition are materialized correctly. A materialized condition // should yield `1` if it evaluated to true, and `0` otherwise. // We force the materialization of comparisons for different combinations of @@ -689,11 +758,11 @@ TEST(CodegenTest, MaterializedCondition1) { block->InsertInstructionBefore(move, block->GetLastInstruction()); }; - RunCodeOptimized(graph, hook_before_codegen, true, lhs[i] < rhs[i]); + RunCodeOptimized(GetParam(), graph, hook_before_codegen, true, lhs[i] < rhs[i]); } } -TEST(CodegenTest, MaterializedCondition2) { +TEST_P(CodegenTest, MaterializedCondition2) { // Check that HIf correctly interprets a materialized condition. // We force the materialization of comparisons for different combinations of // inputs. An HIf takes the materialized combination as input and returns a @@ -755,31 +824,35 @@ TEST(CodegenTest, MaterializedCondition2) { block->InsertInstructionBefore(move, block->GetLastInstruction()); }; - RunCodeOptimized(graph, hook_before_codegen, true, lhs[i] < rhs[i]); + RunCodeOptimized(GetParam(), graph, hook_before_codegen, true, lhs[i] < rhs[i]); } } -TEST(CodegenTest, ReturnDivIntLit8) { +TEST_P(CodegenTest, ReturnDivIntLit8) { const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 4 << 12 | 0 << 8, Instruction::DIV_INT_LIT8, 3 << 8 | 0, Instruction::RETURN); - TestCode(data, true, 1); + TestCode(GetParam(), data, true, 1); } -TEST(CodegenTest, ReturnDivInt2Addr) { +TEST_P(CodegenTest, ReturnDivInt2Addr) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 4 << 12 | 0, Instruction::CONST_4 | 2 << 12 | 1 << 8, Instruction::DIV_INT_2ADDR | 1 << 12, Instruction::RETURN); - TestCode(data, true, 2); + TestCode(GetParam(), data, true, 2); } // Helper method. -static void TestComparison(IfCondition condition, int64_t i, int64_t j, Primitive::Type type) { +static void TestComparison(IfCondition condition, + int64_t i, + int64_t j, + Primitive::Type type, + const InstructionSet target_isa) { ArenaPool pool; ArenaAllocator allocator(&pool); HGraph* graph = CreateGraph(&allocator); @@ -862,46 +935,78 @@ static void TestComparison(IfCondition condition, int64_t i, int64_t j, Primitiv auto hook_before_codegen = [](HGraph*) { }; - RunCodeOptimized(graph, hook_before_codegen, true, expected_result); + RunCodeOptimized(target_isa, graph, hook_before_codegen, true, expected_result); } -TEST(CodegenTest, ComparisonsInt) { +TEST_P(CodegenTest, ComparisonsInt) { + const InstructionSet target_isa = GetParam(); for (int64_t i = -1; i <= 1; i++) { for (int64_t j = -1; j <= 1; j++) { - TestComparison(kCondEQ, i, j, Primitive::kPrimInt); - TestComparison(kCondNE, i, j, Primitive::kPrimInt); - TestComparison(kCondLT, i, j, Primitive::kPrimInt); - TestComparison(kCondLE, i, j, Primitive::kPrimInt); - TestComparison(kCondGT, i, j, Primitive::kPrimInt); - TestComparison(kCondGE, i, j, Primitive::kPrimInt); - TestComparison(kCondB, i, j, Primitive::kPrimInt); - TestComparison(kCondBE, i, j, Primitive::kPrimInt); - TestComparison(kCondA, i, j, Primitive::kPrimInt); - TestComparison(kCondAE, i, j, Primitive::kPrimInt); + TestComparison(kCondEQ, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondNE, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondLT, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondLE, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondGT, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondGE, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondB, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondBE, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondA, i, j, Primitive::kPrimInt, target_isa); + TestComparison(kCondAE, i, j, Primitive::kPrimInt, target_isa); } } } -TEST(CodegenTest, ComparisonsLong) { +TEST_P(CodegenTest, ComparisonsLong) { // TODO: make MIPS work for long if (kRuntimeISA == kMips || kRuntimeISA == kMips64) { return; } + const InstructionSet target_isa = GetParam(); + if (target_isa == kMips || target_isa == kMips64) { + return; + } + for (int64_t i = -1; i <= 1; i++) { for (int64_t j = -1; j <= 1; j++) { - TestComparison(kCondEQ, i, j, Primitive::kPrimLong); - TestComparison(kCondNE, i, j, Primitive::kPrimLong); - TestComparison(kCondLT, i, j, Primitive::kPrimLong); - TestComparison(kCondLE, i, j, Primitive::kPrimLong); - TestComparison(kCondGT, i, j, Primitive::kPrimLong); - TestComparison(kCondGE, i, j, Primitive::kPrimLong); - TestComparison(kCondB, i, j, Primitive::kPrimLong); - TestComparison(kCondBE, i, j, Primitive::kPrimLong); - TestComparison(kCondA, i, j, Primitive::kPrimLong); - TestComparison(kCondAE, i, j, Primitive::kPrimLong); + TestComparison(kCondEQ, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondNE, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondLT, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondLE, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondGT, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondGE, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondB, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondBE, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondA, i, j, Primitive::kPrimLong, target_isa); + TestComparison(kCondAE, i, j, Primitive::kPrimLong, target_isa); } } } +static ::std::vector<InstructionSet> GetTargetISAs() { + ::std::vector<InstructionSet> v; + // Add all ISAs that are executable on hardware or on simulator. + const ::std::vector<InstructionSet> executable_isa_candidates = { + kArm, + kArm64, + kThumb2, + kX86, + kX86_64, + kMips, + kMips64 + }; + + for (auto target_isa : executable_isa_candidates) { + if (CanExecute(target_isa)) { + v.push_back(target_isa); + } + } + + return v; +} + +INSTANTIATE_TEST_CASE_P(MultipleTargets, + CodegenTest, + ::testing::ValuesIn(GetTargetISAs())); + } // namespace art diff --git a/compiler/optimizing/induction_var_analysis.cc b/compiler/optimizing/induction_var_analysis.cc index 0b7fdf85ea..19e6cbd314 100644 --- a/compiler/optimizing/induction_var_analysis.cc +++ b/compiler/optimizing/induction_var_analysis.cc @@ -71,10 +71,10 @@ HInductionVarAnalysis::HInductionVarAnalysis(HGraph* graph) } void HInductionVarAnalysis::Run() { - // Detects sequence variables (generalized induction variables) during an inner-loop-first - // traversal of all loops using Gerlek's algorithm. The order is only relevant if outer - // loops would use induction information of inner loops (not currently done). - for (HPostOrderIterator it_graph(*graph_); !it_graph.Done(); it_graph.Advance()) { + // Detects sequence variables (generalized induction variables) during an outer to inner + // traversal of all loops using Gerlek's algorithm. The order is important to enable + // range analysis on outer loop while visiting inner loops. + for (HReversePostOrderIterator it_graph(*graph_); !it_graph.Done(); it_graph.Advance()) { HBasicBlock* graph_block = it_graph.Current(); if (graph_block->IsLoopHeader()) { VisitLoop(graph_block->GetLoopInformation()); @@ -745,8 +745,7 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::CreateSimplifiedInv if (value == 1) { return b; } else if (value == -1) { - op = kNeg; - a = nullptr; + return CreateSimplifiedInvariant(kNeg, nullptr, b); } } } @@ -763,41 +762,27 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::CreateSimplifiedInv if (value == 1) { return a; } else if (value == -1) { - op = kNeg; - b = a; - a = nullptr; + return CreateSimplifiedInvariant(kNeg, nullptr, a); } } } else if (b->operation == kNeg) { // Simplify a + (-b) = a - b, a - (-b) = a + b, -(-b) = b. if (op == kAdd) { - op = kSub; - b = b->op_b; + return CreateSimplifiedInvariant(kSub, a, b->op_b); } else if (op == kSub) { - op = kAdd; - b = b->op_b; + return CreateSimplifiedInvariant(kAdd, a, b->op_b); } else if (op == kNeg) { return b->op_b; } + } else if (b->operation == kSub) { + // Simplify - (a - b) = b - a. + if (op == kNeg) { + return CreateSimplifiedInvariant(kSub, b->op_b, b->op_a); + } } return new (graph_->GetArena()) InductionInfo(kInvariant, op, a, b, nullptr); } -bool HInductionVarAnalysis::InductionEqual(InductionInfo* info1, - InductionInfo* info2) { - // Test structural equality only, without accounting for simplifications. - if (info1 != nullptr && info2 != nullptr) { - return - info1->induction_class == info2->induction_class && - info1->operation == info2->operation && - info1->fetch == info2->fetch && - InductionEqual(info1->op_a, info2->op_a) && - InductionEqual(info1->op_b, info2->op_b); - } - // Otherwise only two nullptrs are considered equal. - return info1 == info2; -} - bool HInductionVarAnalysis::IsIntAndGet(InductionInfo* info, int64_t* value) { if (info != nullptr && info->induction_class == kInvariant) { // A direct constant fetch. @@ -812,19 +797,35 @@ bool HInductionVarAnalysis::IsIntAndGet(InductionInfo* info, int64_t* value) { } } // Use range analysis to resolve compound values. - int32_t range_value; - if (InductionVarRange::GetConstant(info, &range_value)) { - *value = range_value; + InductionVarRange range(this); + int32_t min_val = 0; + int32_t max_val = 0; + if (range.IsConstantRange(info, &min_val, &max_val) && min_val == max_val) { + *value = min_val; return true; } } return false; } +bool HInductionVarAnalysis::InductionEqual(InductionInfo* info1, + InductionInfo* info2) { + // Test structural equality only, without accounting for simplifications. + if (info1 != nullptr && info2 != nullptr) { + return + info1->induction_class == info2->induction_class && + info1->operation == info2->operation && + info1->fetch == info2->fetch && + InductionEqual(info1->op_a, info2->op_a) && + InductionEqual(info1->op_b, info2->op_b); + } + // Otherwise only two nullptrs are considered equal. + return info1 == info2; +} + std::string HInductionVarAnalysis::InductionToString(InductionInfo* info) { if (info != nullptr) { if (info->induction_class == kInvariant) { - int64_t value = -1; std::string inv = "("; inv += InductionToString(info->op_a); switch (info->operation) { @@ -840,8 +841,10 @@ std::string HInductionVarAnalysis::InductionToString(InductionInfo* info) { case kGE: inv += " >= "; break; case kFetch: DCHECK(info->fetch); - if (IsIntAndGet(info, &value)) { - inv += std::to_string(value); + if (info->fetch->IsIntConstant()) { + inv += std::to_string(info->fetch->AsIntConstant()->GetValue()); + } else if (info->fetch->IsLongConstant()) { + inv += std::to_string(info->fetch->AsLongConstant()->GetValue()); } else { inv += std::to_string(info->fetch->GetId()) + ":" + info->fetch->DebugName(); } diff --git a/compiler/optimizing/induction_var_analysis.h b/compiler/optimizing/induction_var_analysis.h index cf354093f2..84d5d82568 100644 --- a/compiler/optimizing/induction_var_analysis.h +++ b/compiler/optimizing/induction_var_analysis.h @@ -188,9 +188,11 @@ class HInductionVarAnalysis : public HOptimization { InductionInfo* CreateConstant(int64_t value, Primitive::Type type); InductionInfo* CreateSimplifiedInvariant(InductionOp op, InductionInfo* a, InductionInfo* b); + // Constants. + bool IsIntAndGet(InductionInfo* info, int64_t* value); + // Helpers. static bool InductionEqual(InductionInfo* info1, InductionInfo* info2); - static bool IsIntAndGet(InductionInfo* info, int64_t* value); static std::string InductionToString(InductionInfo* info); // TODO: fine tune the following data structures, only keep relevant data. diff --git a/compiler/optimizing/induction_var_range.cc b/compiler/optimizing/induction_var_range.cc index 9d0cde7c9f..ae15fcf381 100644 --- a/compiler/optimizing/induction_var_range.cc +++ b/compiler/optimizing/induction_var_range.cc @@ -119,7 +119,7 @@ void InductionVarRange::GetInductionRange(HInstruction* context, } } -bool InductionVarRange::RefineOuter(/*in-out*/Value* min_val, /*in-out*/Value* max_val) { +bool InductionVarRange::RefineOuter(/*in-out*/Value* min_val, /*in-out*/Value* max_val) const { Value v1 = RefineOuter(*min_val, /* is_min */ true); Value v2 = RefineOuter(*max_val, /* is_min */ false); if (v1.instruction != min_val->instruction || v2.instruction != max_val->instruction) { @@ -167,7 +167,7 @@ void InductionVarRange::GenerateTakenTest(HInstruction* context, // Private class methods. // -bool InductionVarRange::NeedsTripCount(HInductionVarAnalysis::InductionInfo* info) { +bool InductionVarRange::NeedsTripCount(HInductionVarAnalysis::InductionInfo* info) const { if (info != nullptr) { if (info->induction_class == HInductionVarAnalysis::kLinear) { return true; @@ -178,7 +178,7 @@ bool InductionVarRange::NeedsTripCount(HInductionVarAnalysis::InductionInfo* inf return false; } -bool InductionVarRange::IsBodyTripCount(HInductionVarAnalysis::InductionInfo* trip) { +bool InductionVarRange::IsBodyTripCount(HInductionVarAnalysis::InductionInfo* trip) const { if (trip != nullptr) { if (trip->induction_class == HInductionVarAnalysis::kInvariant) { return trip->operation == HInductionVarAnalysis::kTripCountInBody || @@ -188,7 +188,7 @@ bool InductionVarRange::IsBodyTripCount(HInductionVarAnalysis::InductionInfo* tr return false; } -bool InductionVarRange::IsUnsafeTripCount(HInductionVarAnalysis::InductionInfo* trip) { +bool InductionVarRange::IsUnsafeTripCount(HInductionVarAnalysis::InductionInfo* trip) const { if (trip != nullptr) { if (trip->induction_class == HInductionVarAnalysis::kInvariant) { return trip->operation == HInductionVarAnalysis::kTripCountInBodyUnsafe || @@ -198,10 +198,57 @@ bool InductionVarRange::IsUnsafeTripCount(HInductionVarAnalysis::InductionInfo* return false; } +InductionVarRange::Value InductionVarRange::GetLinear(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const { + // Detect common situation where an offset inside the trip count cancels out during range + // analysis (finding max a * (TC - 1) + OFFSET for a == 1 and TC = UPPER - OFFSET or finding + // min a * (TC - 1) + OFFSET for a == -1 and TC = OFFSET - UPPER) to avoid losing information + // with intermediate results that only incorporate single instructions. + if (trip != nullptr) { + HInductionVarAnalysis::InductionInfo* trip_expr = trip->op_a; + if (trip_expr->operation == HInductionVarAnalysis::kSub) { + int32_t min_value = 0; + int32_t stride_value = 0; + if (IsConstantRange(info->op_a, &min_value, &stride_value) && min_value == stride_value) { + if (!is_min && stride_value == 1) { + // Test original trip's negative operand (trip_expr->op_b) against + // the offset of the linear induction. + if (HInductionVarAnalysis::InductionEqual(trip_expr->op_b, info->op_b)) { + // Analyze cancelled trip with just the positive operand (trip_expr->op_a). + HInductionVarAnalysis::InductionInfo cancelled_trip( + trip->induction_class, trip->operation, trip_expr->op_a, trip->op_b, nullptr); + return GetVal(&cancelled_trip, trip, in_body, is_min); + } + } else if (is_min && stride_value == -1) { + // Test original trip's positive operand (trip_expr->op_a) against + // the offset of the linear induction. + if (HInductionVarAnalysis::InductionEqual(trip_expr->op_a, info->op_b)) { + // Analyze cancelled trip with just the negative operand (trip_expr->op_b). + HInductionVarAnalysis::InductionInfo neg( + HInductionVarAnalysis::kInvariant, + HInductionVarAnalysis::kNeg, + nullptr, + trip_expr->op_b, + nullptr); + HInductionVarAnalysis::InductionInfo cancelled_trip( + trip->induction_class, trip->operation, &neg, trip->op_b, nullptr); + return SubValue(Value(0), GetVal(&cancelled_trip, trip, in_body, !is_min)); + } + } + } + } + } + // General rule of linear induction a * i + b, for normalized 0 <= i < TC. + return AddValue(GetMul(info->op_a, trip, trip, in_body, is_min), + GetVal(info->op_b, trip, in_body, is_min)); +} + InductionVarRange::Value InductionVarRange::GetFetch(HInstruction* instruction, HInductionVarAnalysis::InductionInfo* trip, bool in_body, - bool is_min) { + bool is_min) const { // Detect constants and chase the fetch a bit deeper into the HIR tree, so that it becomes // more likely range analysis will compare the same instructions as terminal nodes. int32_t value; @@ -227,7 +274,7 @@ InductionVarRange::Value InductionVarRange::GetFetch(HInstruction* instruction, InductionVarRange::Value InductionVarRange::GetVal(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* trip, bool in_body, - bool is_min) { + bool is_min) const { if (info != nullptr) { switch (info->induction_class) { case HInductionVarAnalysis::kInvariant: @@ -266,13 +313,11 @@ InductionVarRange::Value InductionVarRange::GetVal(HInductionVarAnalysis::Induct break; } break; - case HInductionVarAnalysis::kLinear: - // Linear induction a * i + b, for normalized 0 <= i < TC. - return AddValue(GetMul(info->op_a, trip, trip, in_body, is_min), - GetVal(info->op_b, trip, in_body, is_min)); + case HInductionVarAnalysis::kLinear: { + return GetLinear(info, trip, in_body, is_min); + } case HInductionVarAnalysis::kWrapAround: case HInductionVarAnalysis::kPeriodic: - // Merge values in the wrap-around/periodic. return MergeVal(GetVal(info->op_a, trip, in_body, is_min), GetVal(info->op_b, trip, in_body, is_min), is_min); } @@ -284,11 +329,17 @@ InductionVarRange::Value InductionVarRange::GetMul(HInductionVarAnalysis::Induct HInductionVarAnalysis::InductionInfo* info2, HInductionVarAnalysis::InductionInfo* trip, bool in_body, - bool is_min) { + bool is_min) const { Value v1_min = GetVal(info1, trip, in_body, /* is_min */ true); Value v1_max = GetVal(info1, trip, in_body, /* is_min */ false); Value v2_min = GetVal(info2, trip, in_body, /* is_min */ true); Value v2_max = GetVal(info2, trip, in_body, /* is_min */ false); + // Try to refine certain failure. + if (v1_min.a_constant && v1_max.a_constant) { + v1_min = RefineOuter(v1_min, /* is_min */ true); + v1_max = RefineOuter(v1_max, /* is_min */ false); + } + // Positive or negative range? if (v1_min.is_known && v1_min.a_constant == 0 && v1_min.b_constant >= 0) { // Positive range vs. positive or negative range. if (v2_min.is_known && v2_min.a_constant == 0 && v2_min.b_constant >= 0) { @@ -298,7 +349,7 @@ InductionVarRange::Value InductionVarRange::GetMul(HInductionVarAnalysis::Induct return is_min ? MulValue(v1_max, v2_min) : MulValue(v1_min, v2_max); } - } else if (v1_min.is_known && v1_min.a_constant == 0 && v1_min.b_constant <= 0) { + } else if (v1_max.is_known && v1_max.a_constant == 0 && v1_max.b_constant <= 0) { // Negative range vs. positive or negative range. if (v2_min.is_known && v2_min.a_constant == 0 && v2_min.b_constant >= 0) { return is_min ? MulValue(v1_min, v2_max) @@ -315,11 +366,12 @@ InductionVarRange::Value InductionVarRange::GetDiv(HInductionVarAnalysis::Induct HInductionVarAnalysis::InductionInfo* info2, HInductionVarAnalysis::InductionInfo* trip, bool in_body, - bool is_min) { + bool is_min) const { Value v1_min = GetVal(info1, trip, in_body, /* is_min */ true); Value v1_max = GetVal(info1, trip, in_body, /* is_min */ false); Value v2_min = GetVal(info2, trip, in_body, /* is_min */ true); Value v2_max = GetVal(info2, trip, in_body, /* is_min */ false); + // Positive or negative range? if (v1_min.is_known && v1_min.a_constant == 0 && v1_min.b_constant >= 0) { // Positive range vs. positive or negative range. if (v2_min.is_known && v2_min.a_constant == 0 && v2_min.b_constant >= 0) { @@ -329,7 +381,7 @@ InductionVarRange::Value InductionVarRange::GetDiv(HInductionVarAnalysis::Induct return is_min ? DivValue(v1_max, v2_max) : DivValue(v1_min, v2_min); } - } else if (v1_min.is_known && v1_min.a_constant == 0 && v1_min.b_constant <= 0) { + } else if (v1_max.is_known && v1_max.a_constant == 0 && v1_max.b_constant <= 0) { // Negative range vs. positive or negative range. if (v2_min.is_known && v2_min.a_constant == 0 && v2_min.b_constant >= 0) { return is_min ? DivValue(v1_min, v2_min) @@ -342,19 +394,23 @@ InductionVarRange::Value InductionVarRange::GetDiv(HInductionVarAnalysis::Induct return Value(); } -bool InductionVarRange::GetConstant(HInductionVarAnalysis::InductionInfo* info, int32_t *value) { - Value v_min = GetVal(info, nullptr, false, /* is_min */ true); - Value v_max = GetVal(info, nullptr, false, /* is_min */ false); - if (v_min.is_known && v_max.is_known) { - if (v_min.a_constant == 0 && v_max.a_constant == 0 && v_min.b_constant == v_max.b_constant) { - *value = v_min.b_constant; +bool InductionVarRange::IsConstantRange(HInductionVarAnalysis::InductionInfo* info, + int32_t *min_value, + int32_t *max_value) const { + bool in_body = true; // no known trip count + Value v_min = GetVal(info, nullptr, in_body, /* is_min */ true); + Value v_max = GetVal(info, nullptr, in_body, /* is_min */ false); + do { + if (v_min.is_known && v_min.a_constant == 0 && v_max.is_known && v_max.a_constant == 0) { + *min_value = v_min.b_constant; + *max_value = v_max.b_constant; return true; } - } + } while (RefineOuter(&v_min, &v_max)); return false; } -InductionVarRange::Value InductionVarRange::AddValue(Value v1, Value v2) { +InductionVarRange::Value InductionVarRange::AddValue(Value v1, Value v2) const { if (v1.is_known && v2.is_known && IsSafeAdd(v1.b_constant, v2.b_constant)) { const int32_t b = v1.b_constant + v2.b_constant; if (v1.a_constant == 0) { @@ -368,7 +424,7 @@ InductionVarRange::Value InductionVarRange::AddValue(Value v1, Value v2) { return Value(); } -InductionVarRange::Value InductionVarRange::SubValue(Value v1, Value v2) { +InductionVarRange::Value InductionVarRange::SubValue(Value v1, Value v2) const { if (v1.is_known && v2.is_known && IsSafeSub(v1.b_constant, v2.b_constant)) { const int32_t b = v1.b_constant - v2.b_constant; if (v1.a_constant == 0 && IsSafeSub(0, v2.a_constant)) { @@ -382,7 +438,7 @@ InductionVarRange::Value InductionVarRange::SubValue(Value v1, Value v2) { return Value(); } -InductionVarRange::Value InductionVarRange::MulValue(Value v1, Value v2) { +InductionVarRange::Value InductionVarRange::MulValue(Value v1, Value v2) const { if (v1.is_known && v2.is_known) { if (v1.a_constant == 0) { if (IsSafeMul(v1.b_constant, v2.a_constant) && IsSafeMul(v1.b_constant, v2.b_constant)) { @@ -397,7 +453,7 @@ InductionVarRange::Value InductionVarRange::MulValue(Value v1, Value v2) { return Value(); } -InductionVarRange::Value InductionVarRange::DivValue(Value v1, Value v2) { +InductionVarRange::Value InductionVarRange::DivValue(Value v1, Value v2) const { if (v1.is_known && v2.is_known && v1.a_constant == 0 && v2.a_constant == 0) { if (IsSafeDiv(v1.b_constant, v2.b_constant)) { return Value(v1.b_constant / v2.b_constant); @@ -406,7 +462,7 @@ InductionVarRange::Value InductionVarRange::DivValue(Value v1, Value v2) { return Value(); } -InductionVarRange::Value InductionVarRange::MergeVal(Value v1, Value v2, bool is_min) { +InductionVarRange::Value InductionVarRange::MergeVal(Value v1, Value v2, bool is_min) const { if (v1.is_known && v2.is_known) { if (v1.instruction == v2.instruction && v1.a_constant == v2.a_constant) { return Value(v1.instruction, v1.a_constant, @@ -417,7 +473,7 @@ InductionVarRange::Value InductionVarRange::MergeVal(Value v1, Value v2, bool is return Value(); } -InductionVarRange::Value InductionVarRange::RefineOuter(Value v, bool is_min) { +InductionVarRange::Value InductionVarRange::RefineOuter(Value v, bool is_min) const { if (v.instruction != nullptr) { HLoopInformation* loop = v.instruction->GetBlock()->GetLoopInformation(); // closest enveloping loop @@ -444,7 +500,7 @@ bool InductionVarRange::GenerateCode(HInstruction* context, /*out*/HInstruction** upper, /*out*/HInstruction** taken_test, /*out*/bool* needs_finite_test, - /*out*/bool* needs_taken_test) { + /*out*/bool* needs_taken_test) const { HLoopInformation* loop = context->GetBlock()->GetLoopInformation(); // closest enveloping loop if (loop != nullptr) { // Set up loop information. @@ -492,7 +548,7 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, HBasicBlock* block, /*out*/HInstruction** result, bool in_body, - bool is_min) { + bool is_min) const { if (info != nullptr) { // Handle current operation. Primitive::Type type = Primitive::kPrimInt; @@ -586,8 +642,9 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, case HInductionVarAnalysis::kLinear: { // Linear induction a * i + b, for normalized 0 <= i < TC. Restrict to unit stride only // to avoid arithmetic wrap-around situations that are hard to guard against. + int32_t min_value = 0; int32_t stride_value = 0; - if (GetConstant(info->op_a, &stride_value)) { + if (IsConstantRange(info->op_a, &min_value, &stride_value) && min_value == stride_value) { if (stride_value == 1 || stride_value == -1) { const bool is_min_a = stride_value == 1 ? is_min : !is_min; if (GenerateCode(trip, trip, graph, block, &opa, in_body, is_min_a) && diff --git a/compiler/optimizing/induction_var_range.h b/compiler/optimizing/induction_var_range.h index 71b0b1b4c3..974b8fba06 100644 --- a/compiler/optimizing/induction_var_range.h +++ b/compiler/optimizing/induction_var_range.h @@ -69,7 +69,7 @@ class InductionVarRange { /*out*/bool* needs_finite_test); /** Refines the values with induction of next outer loop. Returns true on change. */ - bool RefineOuter(/*in-out*/Value* min_val, /*in-out*/Value* max_val); + bool RefineOuter(/*in-out*/Value* min_val, /*in-out*/Value* max_val) const; /** * Returns true if range analysis is able to generate code for the lower and upper @@ -116,46 +116,48 @@ class InductionVarRange { /*out*/HInstruction** taken_test); private: - // - // Private helper methods. - // - - static bool NeedsTripCount(HInductionVarAnalysis::InductionInfo* info); - static bool IsBodyTripCount(HInductionVarAnalysis::InductionInfo* trip); - static bool IsUnsafeTripCount(HInductionVarAnalysis::InductionInfo* trip); - - static Value GetFetch(HInstruction* instruction, - HInductionVarAnalysis::InductionInfo* trip, - bool in_body, - bool is_min); - static Value GetVal(HInductionVarAnalysis::InductionInfo* info, - HInductionVarAnalysis::InductionInfo* trip, - bool in_body, - bool is_min); - static Value GetMul(HInductionVarAnalysis::InductionInfo* info1, - HInductionVarAnalysis::InductionInfo* info2, - HInductionVarAnalysis::InductionInfo* trip, - bool in_body, - bool is_min); - static Value GetDiv(HInductionVarAnalysis::InductionInfo* info1, - HInductionVarAnalysis::InductionInfo* info2, - HInductionVarAnalysis::InductionInfo* trip, - bool in_body, - bool is_min); - - static bool GetConstant(HInductionVarAnalysis::InductionInfo* info, int32_t *value); - - static Value AddValue(Value v1, Value v2); - static Value SubValue(Value v1, Value v2); - static Value MulValue(Value v1, Value v2); - static Value DivValue(Value v1, Value v2); - static Value MergeVal(Value v1, Value v2, bool is_min); + bool NeedsTripCount(HInductionVarAnalysis::InductionInfo* info) const; + bool IsBodyTripCount(HInductionVarAnalysis::InductionInfo* trip) const; + bool IsUnsafeTripCount(HInductionVarAnalysis::InductionInfo* trip) const; + + Value GetLinear(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const; + Value GetFetch(HInstruction* instruction, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const; + Value GetVal(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const; + Value GetMul(HInductionVarAnalysis::InductionInfo* info1, + HInductionVarAnalysis::InductionInfo* info2, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const; + Value GetDiv(HInductionVarAnalysis::InductionInfo* info1, + HInductionVarAnalysis::InductionInfo* info2, + HInductionVarAnalysis::InductionInfo* trip, + bool in_body, + bool is_min) const; + + bool IsConstantRange(HInductionVarAnalysis::InductionInfo* info, + int32_t *min_value, + int32_t *max_value) const; + + Value AddValue(Value v1, Value v2) const; + Value SubValue(Value v1, Value v2) const; + Value MulValue(Value v1, Value v2) const; + Value DivValue(Value v1, Value v2) const; + Value MergeVal(Value v1, Value v2, bool is_min) const; /** * Returns refined value using induction of next outer loop or the input value if no * further refinement is possible. */ - Value RefineOuter(Value val, bool is_min); + Value RefineOuter(Value val, bool is_min) const; /** * Generates code for lower/upper/taken-test in the HIR. Returns true on success. @@ -170,15 +172,15 @@ class InductionVarRange { /*out*/HInstruction** upper, /*out*/HInstruction** taken_test, /*out*/bool* needs_finite_test, - /*out*/bool* needs_taken_test); - - static bool GenerateCode(HInductionVarAnalysis::InductionInfo* info, - HInductionVarAnalysis::InductionInfo* trip, - HGraph* graph, - HBasicBlock* block, - /*out*/HInstruction** result, - bool in_body, - bool is_min); + /*out*/bool* needs_taken_test) const; + + bool GenerateCode(HInductionVarAnalysis::InductionInfo* info, + HInductionVarAnalysis::InductionInfo* trip, + HGraph* graph, + HBasicBlock* block, + /*out*/HInstruction** result, + bool in_body, + bool is_min) const; /** Results of prior induction variable analysis. */ HInductionVarAnalysis *induction_analysis_; diff --git a/compiler/optimizing/induction_var_range_test.cc b/compiler/optimizing/induction_var_range_test.cc index 128b5bb811..5c0bdd7c4c 100644 --- a/compiler/optimizing/induction_var_range_test.cc +++ b/compiler/optimizing/induction_var_range_test.cc @@ -31,9 +31,12 @@ using Value = InductionVarRange::Value; */ class InductionVarRangeTest : public testing::Test { public: - InductionVarRangeTest() : pool_(), allocator_(&pool_) { - graph_ = CreateGraph(&allocator_); - iva_ = new (&allocator_) HInductionVarAnalysis(graph_); + InductionVarRangeTest() + : pool_(), + allocator_(&pool_), + graph_(CreateGraph(&allocator_)), + iva_(new (&allocator_) HInductionVarAnalysis(graph_)), + range_(iva_) { BuildGraph(); } @@ -59,6 +62,11 @@ class InductionVarRangeTest : public testing::Test { graph_->AddBlock(exit_block_); graph_->SetEntryBlock(entry_block_); graph_->SetExitBlock(exit_block_); + // Two parameters. + x_ = new (&allocator_) HParameterValue(graph_->GetDexFile(), 0, 0, Primitive::kPrimInt); + entry_block_->AddInstruction(x_); + y_ = new (&allocator_) HParameterValue(graph_->GetDexFile(), 0, 0, Primitive::kPrimInt); + entry_block_->AddInstruction(y_); } /** Constructs loop with given upper bound. */ @@ -103,7 +111,7 @@ class InductionVarRangeTest : public testing::Test { exit_block_->AddInstruction(new (&allocator_) HExit()); } - /** Performs induction variable analysis. */ + /** Constructs SSA and performs induction variable analysis. */ void PerformInductionVarAnalysis() { ASSERT_TRUE(graph_->TryBuildingSsa()); iva_->Run(); @@ -180,49 +188,51 @@ class InductionVarRangeTest : public testing::Test { // bool NeedsTripCount(HInductionVarAnalysis::InductionInfo* info) { - return InductionVarRange::NeedsTripCount(info); + return range_.NeedsTripCount(info); } bool IsBodyTripCount(HInductionVarAnalysis::InductionInfo* trip) { - return InductionVarRange::IsBodyTripCount(trip); + return range_.IsBodyTripCount(trip); } bool IsUnsafeTripCount(HInductionVarAnalysis::InductionInfo* trip) { - return InductionVarRange::IsUnsafeTripCount(trip); + return range_.IsUnsafeTripCount(trip); } Value GetMin(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* induc) { - return InductionVarRange::GetVal(info, induc, /* in_body */ true, /* is_min */ true); + return range_.GetVal(info, induc, /* in_body */ true, /* is_min */ true); } Value GetMax(HInductionVarAnalysis::InductionInfo* info, HInductionVarAnalysis::InductionInfo* induc) { - return InductionVarRange::GetVal(info, induc, /* in_body */ true, /* is_min */ false); + return range_.GetVal(info, induc, /* in_body */ true, /* is_min */ false); } Value GetMul(HInductionVarAnalysis::InductionInfo* info1, HInductionVarAnalysis::InductionInfo* info2, bool is_min) { - return InductionVarRange::GetMul(info1, info2, nullptr, /* in_body */ true, is_min); + return range_.GetMul(info1, info2, nullptr, /* in_body */ true, is_min); } Value GetDiv(HInductionVarAnalysis::InductionInfo* info1, HInductionVarAnalysis::InductionInfo* info2, bool is_min) { - return InductionVarRange::GetDiv(info1, info2, nullptr, /* in_body */ true, is_min); + return range_.GetDiv(info1, info2, nullptr, /* in_body */ true, is_min); } - bool GetConstant(HInductionVarAnalysis::InductionInfo* info, int32_t* value) { - return InductionVarRange::GetConstant(info, value); + bool IsConstantRange(HInductionVarAnalysis::InductionInfo* info, + int32_t* min_value, + int32_t* max_value) { + return range_.IsConstantRange(info, min_value, max_value); } - Value AddValue(Value v1, Value v2) { return InductionVarRange::AddValue(v1, v2); } - Value SubValue(Value v1, Value v2) { return InductionVarRange::SubValue(v1, v2); } - Value MulValue(Value v1, Value v2) { return InductionVarRange::MulValue(v1, v2); } - Value DivValue(Value v1, Value v2) { return InductionVarRange::DivValue(v1, v2); } - Value MinValue(Value v1, Value v2) { return InductionVarRange::MergeVal(v1, v2, true); } - Value MaxValue(Value v1, Value v2) { return InductionVarRange::MergeVal(v1, v2, false); } + Value AddValue(Value v1, Value v2) { return range_.AddValue(v1, v2); } + Value SubValue(Value v1, Value v2) { return range_.SubValue(v1, v2); } + Value MulValue(Value v1, Value v2) { return range_.MulValue(v1, v2); } + Value DivValue(Value v1, Value v2) { return range_.DivValue(v1, v2); } + Value MinValue(Value v1, Value v2) { return range_.MergeVal(v1, v2, true); } + Value MaxValue(Value v1, Value v2) { return range_.MergeVal(v1, v2, false); } // General building fields. ArenaPool pool_; @@ -232,16 +242,17 @@ class InductionVarRangeTest : public testing::Test { HBasicBlock* exit_block_; HBasicBlock* loop_preheader_; HInductionVarAnalysis* iva_; + InductionVarRange range_; // Instructions. HInstruction* condition_; HInstruction* increment_; - HReturnVoid x_; - HReturnVoid y_; + HInstruction* x_; + HInstruction* y_; }; // -// Tests on static methods. +// Tests on private methods. // TEST_F(InductionVarRangeTest, TripCountProperties) { @@ -274,14 +285,14 @@ TEST_F(InductionVarRangeTest, GetMinMaxAdd) { GetMin(CreateInvariant('+', CreateConst(2), CreateRange(10, 20)), nullptr)); ExpectEqual(Value(22), GetMax(CreateInvariant('+', CreateConst(2), CreateRange(10, 20)), nullptr)); - ExpectEqual(Value(&x_, 1, -20), - GetMin(CreateInvariant('+', CreateFetch(&x_), CreateRange(-20, -10)), nullptr)); - ExpectEqual(Value(&x_, 1, -10), - GetMax(CreateInvariant('+', CreateFetch(&x_), CreateRange(-20, -10)), nullptr)); - ExpectEqual(Value(&x_, 1, 10), - GetMin(CreateInvariant('+', CreateRange(10, 20), CreateFetch(&x_)), nullptr)); - ExpectEqual(Value(&x_, 1, 20), - GetMax(CreateInvariant('+', CreateRange(10, 20), CreateFetch(&x_)), nullptr)); + ExpectEqual(Value(x_, 1, -20), + GetMin(CreateInvariant('+', CreateFetch(x_), CreateRange(-20, -10)), nullptr)); + ExpectEqual(Value(x_, 1, -10), + GetMax(CreateInvariant('+', CreateFetch(x_), CreateRange(-20, -10)), nullptr)); + ExpectEqual(Value(x_, 1, 10), + GetMin(CreateInvariant('+', CreateRange(10, 20), CreateFetch(x_)), nullptr)); + ExpectEqual(Value(x_, 1, 20), + GetMax(CreateInvariant('+', CreateRange(10, 20), CreateFetch(x_)), nullptr)); ExpectEqual(Value(5), GetMin(CreateInvariant('+', CreateRange(-5, -1), CreateRange(10, 20)), nullptr)); ExpectEqual(Value(19), @@ -293,14 +304,14 @@ TEST_F(InductionVarRangeTest, GetMinMaxSub) { GetMin(CreateInvariant('-', CreateConst(2), CreateRange(10, 20)), nullptr)); ExpectEqual(Value(-8), GetMax(CreateInvariant('-', CreateConst(2), CreateRange(10, 20)), nullptr)); - ExpectEqual(Value(&x_, 1, 10), - GetMin(CreateInvariant('-', CreateFetch(&x_), CreateRange(-20, -10)), nullptr)); - ExpectEqual(Value(&x_, 1, 20), - GetMax(CreateInvariant('-', CreateFetch(&x_), CreateRange(-20, -10)), nullptr)); - ExpectEqual(Value(&x_, -1, 10), - GetMin(CreateInvariant('-', CreateRange(10, 20), CreateFetch(&x_)), nullptr)); - ExpectEqual(Value(&x_, -1, 20), - GetMax(CreateInvariant('-', CreateRange(10, 20), CreateFetch(&x_)), nullptr)); + ExpectEqual(Value(x_, 1, 10), + GetMin(CreateInvariant('-', CreateFetch(x_), CreateRange(-20, -10)), nullptr)); + ExpectEqual(Value(x_, 1, 20), + GetMax(CreateInvariant('-', CreateFetch(x_), CreateRange(-20, -10)), nullptr)); + ExpectEqual(Value(x_, -1, 10), + GetMin(CreateInvariant('-', CreateRange(10, 20), CreateFetch(x_)), nullptr)); + ExpectEqual(Value(x_, -1, 20), + GetMax(CreateInvariant('-', CreateRange(10, 20), CreateFetch(x_)), nullptr)); ExpectEqual(Value(-25), GetMin(CreateInvariant('-', CreateRange(-5, -1), CreateRange(10, 20)), nullptr)); ExpectEqual(Value(-11), @@ -312,8 +323,8 @@ TEST_F(InductionVarRangeTest, GetMinMaxNeg) { ExpectEqual(Value(-10), GetMax(CreateInvariant('n', nullptr, CreateRange(10, 20)), nullptr)); ExpectEqual(Value(10), GetMin(CreateInvariant('n', nullptr, CreateRange(-20, -10)), nullptr)); ExpectEqual(Value(20), GetMax(CreateInvariant('n', nullptr, CreateRange(-20, -10)), nullptr)); - ExpectEqual(Value(&x_, -1, 0), GetMin(CreateInvariant('n', nullptr, CreateFetch(&x_)), nullptr)); - ExpectEqual(Value(&x_, -1, 0), GetMax(CreateInvariant('n', nullptr, CreateFetch(&x_)), nullptr)); + ExpectEqual(Value(x_, -1, 0), GetMin(CreateInvariant('n', nullptr, CreateFetch(x_)), nullptr)); + ExpectEqual(Value(x_, -1, 0), GetMax(CreateInvariant('n', nullptr, CreateFetch(x_)), nullptr)); } TEST_F(InductionVarRangeTest, GetMinMaxMul) { @@ -336,8 +347,8 @@ TEST_F(InductionVarRangeTest, GetMinMaxConstant) { } TEST_F(InductionVarRangeTest, GetMinMaxFetch) { - ExpectEqual(Value(&x_, 1, 0), GetMin(CreateFetch(&x_), nullptr)); - ExpectEqual(Value(&x_, 1, 0), GetMax(CreateFetch(&x_), nullptr)); + ExpectEqual(Value(x_, 1, 0), GetMin(CreateFetch(x_), nullptr)); + ExpectEqual(Value(x_, 1, 0), GetMax(CreateFetch(x_), nullptr)); } TEST_F(InductionVarRangeTest, GetMinMaxLinear) { @@ -364,45 +375,70 @@ TEST_F(InductionVarRangeTest, GetMinMaxPeriodic) { TEST_F(InductionVarRangeTest, GetMulMin) { ExpectEqual(Value(6), GetMul(CreateRange(2, 10), CreateRange(3, 5), true)); ExpectEqual(Value(-50), GetMul(CreateRange(2, 10), CreateRange(-5, -3), true)); + ExpectEqual(Value(), GetMul(CreateRange(2, 10), CreateRange(-1, 1), true)); ExpectEqual(Value(-50), GetMul(CreateRange(-10, -2), CreateRange(3, 5), true)); ExpectEqual(Value(6), GetMul(CreateRange(-10, -2), CreateRange(-5, -3), true)); + ExpectEqual(Value(), GetMul(CreateRange(-10, -2), CreateRange(-1, 1), true)); + ExpectEqual(Value(), GetMul(CreateRange(-1, 1), CreateRange(2, 10), true)); + ExpectEqual(Value(), GetMul(CreateRange(-1, 1), CreateRange(-10, -2), true)); + ExpectEqual(Value(), GetMul(CreateRange(-1, 1), CreateRange(-1, 1), true)); } TEST_F(InductionVarRangeTest, GetMulMax) { ExpectEqual(Value(50), GetMul(CreateRange(2, 10), CreateRange(3, 5), false)); ExpectEqual(Value(-6), GetMul(CreateRange(2, 10), CreateRange(-5, -3), false)); + ExpectEqual(Value(), GetMul(CreateRange(2, 10), CreateRange(-1, 1), false)); ExpectEqual(Value(-6), GetMul(CreateRange(-10, -2), CreateRange(3, 5), false)); ExpectEqual(Value(50), GetMul(CreateRange(-10, -2), CreateRange(-5, -3), false)); + ExpectEqual(Value(), GetMul(CreateRange(-10, -2), CreateRange(-1, 1), false)); + ExpectEqual(Value(), GetMul(CreateRange(-1, 1), CreateRange(2, 10), false)); + ExpectEqual(Value(), GetMul(CreateRange(-1, 1), CreateRange(-10, -2), false)); + ExpectEqual(Value(), GetMul(CreateRange(-1, 1), CreateRange(-1, 1), false)); } TEST_F(InductionVarRangeTest, GetDivMin) { ExpectEqual(Value(10), GetDiv(CreateRange(40, 1000), CreateRange(2, 4), true)); ExpectEqual(Value(-500), GetDiv(CreateRange(40, 1000), CreateRange(-4, -2), true)); + ExpectEqual(Value(), GetDiv(CreateRange(40, 1000), CreateRange(-1, 1), true)); ExpectEqual(Value(-500), GetDiv(CreateRange(-1000, -40), CreateRange(2, 4), true)); ExpectEqual(Value(10), GetDiv(CreateRange(-1000, -40), CreateRange(-4, -2), true)); + ExpectEqual(Value(), GetDiv(CreateRange(-1000, -40), CreateRange(-1, 1), true)); + ExpectEqual(Value(), GetDiv(CreateRange(-1, 1), CreateRange(40, 1000), true)); + ExpectEqual(Value(), GetDiv(CreateRange(-1, 1), CreateRange(-1000, -40), true)); + ExpectEqual(Value(), GetDiv(CreateRange(-1, 1), CreateRange(-1, 1), true)); } TEST_F(InductionVarRangeTest, GetDivMax) { ExpectEqual(Value(500), GetDiv(CreateRange(40, 1000), CreateRange(2, 4), false)); ExpectEqual(Value(-10), GetDiv(CreateRange(40, 1000), CreateRange(-4, -2), false)); + ExpectEqual(Value(), GetDiv(CreateRange(40, 1000), CreateRange(-1, 1), false)); ExpectEqual(Value(-10), GetDiv(CreateRange(-1000, -40), CreateRange(2, 4), false)); ExpectEqual(Value(500), GetDiv(CreateRange(-1000, -40), CreateRange(-4, -2), false)); + ExpectEqual(Value(), GetDiv(CreateRange(-1000, -40), CreateRange(-1, 1), false)); + ExpectEqual(Value(), GetDiv(CreateRange(-1, 1), CreateRange(40, 1000), false)); + ExpectEqual(Value(), GetDiv(CreateRange(-1, 1), CreateRange(-1000, 40), false)); + ExpectEqual(Value(), GetDiv(CreateRange(-1, 1), CreateRange(-1, 1), false)); } -TEST_F(InductionVarRangeTest, GetConstant) { - int32_t value; - ASSERT_TRUE(GetConstant(CreateConst(12345), &value)); - EXPECT_EQ(12345, value); - EXPECT_FALSE(GetConstant(CreateRange(1, 2), &value)); +TEST_F(InductionVarRangeTest, IsConstantRange) { + int32_t min_value; + int32_t max_value; + ASSERT_TRUE(IsConstantRange(CreateConst(12345), &min_value, &max_value)); + EXPECT_EQ(12345, min_value); + EXPECT_EQ(12345, max_value); + ASSERT_TRUE(IsConstantRange(CreateRange(1, 2), &min_value, &max_value)); + EXPECT_EQ(1, min_value); + EXPECT_EQ(2, max_value); + EXPECT_FALSE(IsConstantRange(CreateFetch(x_), &min_value, &max_value)); } TEST_F(InductionVarRangeTest, AddValue) { ExpectEqual(Value(110), AddValue(Value(10), Value(100))); - ExpectEqual(Value(-5), AddValue(Value(&x_, 1, -4), Value(&x_, -1, -1))); - ExpectEqual(Value(&x_, 3, -5), AddValue(Value(&x_, 2, -4), Value(&x_, 1, -1))); - ExpectEqual(Value(), AddValue(Value(&x_, 1, 5), Value(&y_, 1, -7))); - ExpectEqual(Value(&x_, 1, 23), AddValue(Value(&x_, 1, 20), Value(3))); - ExpectEqual(Value(&y_, 1, 5), AddValue(Value(55), Value(&y_, 1, -50))); + ExpectEqual(Value(-5), AddValue(Value(x_, 1, -4), Value(x_, -1, -1))); + ExpectEqual(Value(x_, 3, -5), AddValue(Value(x_, 2, -4), Value(x_, 1, -1))); + ExpectEqual(Value(), AddValue(Value(x_, 1, 5), Value(y_, 1, -7))); + ExpectEqual(Value(x_, 1, 23), AddValue(Value(x_, 1, 20), Value(3))); + ExpectEqual(Value(y_, 1, 5), AddValue(Value(55), Value(y_, 1, -50))); const int32_t max_value = std::numeric_limits<int32_t>::max(); ExpectEqual(Value(max_value), AddValue(Value(max_value - 5), Value(5))); ExpectEqual(Value(), AddValue(Value(max_value - 5), Value(6))); // unsafe @@ -410,11 +446,11 @@ TEST_F(InductionVarRangeTest, AddValue) { TEST_F(InductionVarRangeTest, SubValue) { ExpectEqual(Value(-90), SubValue(Value(10), Value(100))); - ExpectEqual(Value(-3), SubValue(Value(&x_, 1, -4), Value(&x_, 1, -1))); - ExpectEqual(Value(&x_, 2, -3), SubValue(Value(&x_, 3, -4), Value(&x_, 1, -1))); - ExpectEqual(Value(), SubValue(Value(&x_, 1, 5), Value(&y_, 1, -7))); - ExpectEqual(Value(&x_, 1, 17), SubValue(Value(&x_, 1, 20), Value(3))); - ExpectEqual(Value(&y_, -4, 105), SubValue(Value(55), Value(&y_, 4, -50))); + ExpectEqual(Value(-3), SubValue(Value(x_, 1, -4), Value(x_, 1, -1))); + ExpectEqual(Value(x_, 2, -3), SubValue(Value(x_, 3, -4), Value(x_, 1, -1))); + ExpectEqual(Value(), SubValue(Value(x_, 1, 5), Value(y_, 1, -7))); + ExpectEqual(Value(x_, 1, 17), SubValue(Value(x_, 1, 20), Value(3))); + ExpectEqual(Value(y_, -4, 105), SubValue(Value(55), Value(y_, 4, -50))); const int32_t min_value = std::numeric_limits<int32_t>::min(); ExpectEqual(Value(min_value), SubValue(Value(min_value + 5), Value(5))); ExpectEqual(Value(), SubValue(Value(min_value + 5), Value(6))); // unsafe @@ -422,145 +458,140 @@ TEST_F(InductionVarRangeTest, SubValue) { TEST_F(InductionVarRangeTest, MulValue) { ExpectEqual(Value(1000), MulValue(Value(10), Value(100))); - ExpectEqual(Value(), MulValue(Value(&x_, 1, -4), Value(&x_, 1, -1))); - ExpectEqual(Value(), MulValue(Value(&x_, 1, 5), Value(&y_, 1, -7))); - ExpectEqual(Value(&x_, 9, 60), MulValue(Value(&x_, 3, 20), Value(3))); - ExpectEqual(Value(&y_, 55, -110), MulValue(Value(55), Value(&y_, 1, -2))); + ExpectEqual(Value(), MulValue(Value(x_, 1, -4), Value(x_, 1, -1))); + ExpectEqual(Value(), MulValue(Value(x_, 1, 5), Value(y_, 1, -7))); + ExpectEqual(Value(x_, 9, 60), MulValue(Value(x_, 3, 20), Value(3))); + ExpectEqual(Value(y_, 55, -110), MulValue(Value(55), Value(y_, 1, -2))); ExpectEqual(Value(), MulValue(Value(90000), Value(-90000))); // unsafe } TEST_F(InductionVarRangeTest, DivValue) { ExpectEqual(Value(25), DivValue(Value(100), Value(4))); - ExpectEqual(Value(), DivValue(Value(&x_, 1, -4), Value(&x_, 1, -1))); - ExpectEqual(Value(), DivValue(Value(&x_, 1, 5), Value(&y_, 1, -7))); - ExpectEqual(Value(), DivValue(Value(&x_, 12, 24), Value(3))); - ExpectEqual(Value(), DivValue(Value(55), Value(&y_, 1, -50))); + ExpectEqual(Value(), DivValue(Value(x_, 1, -4), Value(x_, 1, -1))); + ExpectEqual(Value(), DivValue(Value(x_, 1, 5), Value(y_, 1, -7))); + ExpectEqual(Value(), DivValue(Value(x_, 12, 24), Value(3))); + ExpectEqual(Value(), DivValue(Value(55), Value(y_, 1, -50))); ExpectEqual(Value(), DivValue(Value(1), Value(0))); // unsafe } TEST_F(InductionVarRangeTest, MinValue) { ExpectEqual(Value(10), MinValue(Value(10), Value(100))); - ExpectEqual(Value(&x_, 1, -4), MinValue(Value(&x_, 1, -4), Value(&x_, 1, -1))); - ExpectEqual(Value(&x_, 4, -4), MinValue(Value(&x_, 4, -4), Value(&x_, 4, -1))); - ExpectEqual(Value(), MinValue(Value(&x_, 1, 5), Value(&y_, 1, -7))); - ExpectEqual(Value(), MinValue(Value(&x_, 1, 20), Value(3))); - ExpectEqual(Value(), MinValue(Value(55), Value(&y_, 1, -50))); + ExpectEqual(Value(x_, 1, -4), MinValue(Value(x_, 1, -4), Value(x_, 1, -1))); + ExpectEqual(Value(x_, 4, -4), MinValue(Value(x_, 4, -4), Value(x_, 4, -1))); + ExpectEqual(Value(), MinValue(Value(x_, 1, 5), Value(y_, 1, -7))); + ExpectEqual(Value(), MinValue(Value(x_, 1, 20), Value(3))); + ExpectEqual(Value(), MinValue(Value(55), Value(y_, 1, -50))); } TEST_F(InductionVarRangeTest, MaxValue) { ExpectEqual(Value(100), MaxValue(Value(10), Value(100))); - ExpectEqual(Value(&x_, 1, -1), MaxValue(Value(&x_, 1, -4), Value(&x_, 1, -1))); - ExpectEqual(Value(&x_, 4, -1), MaxValue(Value(&x_, 4, -4), Value(&x_, 4, -1))); - ExpectEqual(Value(), MaxValue(Value(&x_, 1, 5), Value(&y_, 1, -7))); - ExpectEqual(Value(), MaxValue(Value(&x_, 1, 20), Value(3))); - ExpectEqual(Value(), MaxValue(Value(55), Value(&y_, 1, -50))); + ExpectEqual(Value(x_, 1, -1), MaxValue(Value(x_, 1, -4), Value(x_, 1, -1))); + ExpectEqual(Value(x_, 4, -1), MaxValue(Value(x_, 4, -4), Value(x_, 4, -1))); + ExpectEqual(Value(), MaxValue(Value(x_, 1, 5), Value(y_, 1, -7))); + ExpectEqual(Value(), MaxValue(Value(x_, 1, 20), Value(3))); + ExpectEqual(Value(), MaxValue(Value(55), Value(y_, 1, -50))); } // -// Tests on instance methods. +// Tests on public methods. // TEST_F(InductionVarRangeTest, ConstantTripCountUp) { BuildLoop(0, graph_->GetIntConstant(1000), 1); PerformInductionVarAnalysis(); - InductionVarRange range(iva_); Value v1, v2; bool needs_finite_test = true; // In context of header: known. - range.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(0), v1); ExpectEqual(Value(1000), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); // In context of loop-body: known. - range.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(0), v1); ExpectEqual(Value(999), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); - range.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); + range_.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(1), v1); ExpectEqual(Value(1000), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); } TEST_F(InductionVarRangeTest, ConstantTripCountDown) { BuildLoop(1000, graph_->GetIntConstant(0), -1); PerformInductionVarAnalysis(); - InductionVarRange range(iva_); Value v1, v2; bool needs_finite_test = true; // In context of header: known. - range.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(0), v1); ExpectEqual(Value(1000), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); // In context of loop-body: known. - range.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(1), v1); ExpectEqual(Value(1000), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); - range.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); + range_.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(0), v1); ExpectEqual(Value(999), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); } TEST_F(InductionVarRangeTest, SymbolicTripCountUp) { - HInstruction* parameter = new (&allocator_) HParameterValue( - graph_->GetDexFile(), 0, 0, Primitive::kPrimInt); - entry_block_->AddInstruction(parameter); - BuildLoop(0, parameter, 1); + BuildLoop(0, x_, 1); PerformInductionVarAnalysis(); - InductionVarRange range(iva_); Value v1, v2; bool needs_finite_test = true; bool needs_taken_test = true; // In context of header: upper unknown. - range.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(0), v1); ExpectEqual(Value(), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); // In context of loop-body: known. - range.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(0), v1); - ExpectEqual(Value(parameter, 1, -1), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); - range.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); + ExpectEqual(Value(x_, 1, -1), v2); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); + range_.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(1), v1); - ExpectEqual(Value(parameter, 1, 0), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + ExpectEqual(Value(x_, 1, 0), v2); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); HInstruction* lower = nullptr; HInstruction* upper = nullptr; HInstruction* taken = nullptr; // Can generate code in context of loop-body only. - EXPECT_FALSE(range.CanGenerateCode( + EXPECT_FALSE(range_.CanGenerateCode( condition_, condition_->InputAt(0), &needs_finite_test, &needs_taken_test)); - ASSERT_TRUE(range.CanGenerateCode( + ASSERT_TRUE(range_.CanGenerateCode( increment_, condition_->InputAt(0), &needs_finite_test, &needs_taken_test)); EXPECT_FALSE(needs_finite_test); EXPECT_TRUE(needs_taken_test); // Generates code. - range.GenerateRangeCode(increment_, condition_->InputAt(0), graph_, loop_preheader_, &lower, &upper); + range_.GenerateRangeCode( + increment_, condition_->InputAt(0), graph_, loop_preheader_, &lower, &upper); // Verify lower is 0+0. ASSERT_TRUE(lower != nullptr); @@ -581,7 +612,7 @@ TEST_F(InductionVarRangeTest, SymbolicTripCountUp) { EXPECT_EQ(0, upper->InputAt(1)->AsIntConstant()->GetValue()); // Verify taken-test is 0<V. - range.GenerateTakenTest(increment_, graph_, loop_preheader_, &taken); + range_.GenerateTakenTest(increment_, graph_, loop_preheader_, &taken); ASSERT_TRUE(taken != nullptr); ASSERT_TRUE(taken->IsLessThan()); ASSERT_TRUE(taken->InputAt(0)->IsIntConstant()); @@ -590,52 +621,49 @@ TEST_F(InductionVarRangeTest, SymbolicTripCountUp) { } TEST_F(InductionVarRangeTest, SymbolicTripCountDown) { - HInstruction* parameter = new (&allocator_) HParameterValue( - graph_->GetDexFile(), 0, 0, Primitive::kPrimInt); - entry_block_->AddInstruction(parameter); - BuildLoop(1000, parameter, -1); + BuildLoop(1000, x_, -1); PerformInductionVarAnalysis(); - InductionVarRange range(iva_); Value v1, v2; bool needs_finite_test = true; bool needs_taken_test = true; // In context of header: lower unknown. - range.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(condition_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); ExpectEqual(Value(), v1); ExpectEqual(Value(1000), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); // In context of loop-body: known. - range.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); + range_.GetInductionRange(increment_, condition_->InputAt(0), &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); - ExpectEqual(Value(parameter, 1, 1), v1); + ExpectEqual(Value(x_, 1, 1), v1); ExpectEqual(Value(1000), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); - range.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); + range_.GetInductionRange(increment_, increment_, &v1, &v2, &needs_finite_test); EXPECT_FALSE(needs_finite_test); - ExpectEqual(Value(parameter, 1, 0), v1); + ExpectEqual(Value(x_, 1, 0), v1); ExpectEqual(Value(999), v2); - EXPECT_FALSE(range.RefineOuter(&v1, &v2)); + EXPECT_FALSE(range_.RefineOuter(&v1, &v2)); HInstruction* lower = nullptr; HInstruction* upper = nullptr; HInstruction* taken = nullptr; // Can generate code in context of loop-body only. - EXPECT_FALSE(range.CanGenerateCode( + EXPECT_FALSE(range_.CanGenerateCode( condition_, condition_->InputAt(0), &needs_finite_test, &needs_taken_test)); - ASSERT_TRUE(range.CanGenerateCode( + ASSERT_TRUE(range_.CanGenerateCode( increment_, condition_->InputAt(0), &needs_finite_test, &needs_taken_test)); EXPECT_FALSE(needs_finite_test); EXPECT_TRUE(needs_taken_test); // Generates code. - range.GenerateRangeCode(increment_, condition_->InputAt(0), graph_, loop_preheader_, &lower, &upper); + range_.GenerateRangeCode( + increment_, condition_->InputAt(0), graph_, loop_preheader_, &lower, &upper); - // Verify lower is 1000-(-(V-1000)-1). + // Verify lower is 1000-((1000-V)-1). ASSERT_TRUE(lower != nullptr); ASSERT_TRUE(lower->IsSub()); ASSERT_TRUE(lower->InputAt(0)->IsIntConstant()); @@ -645,12 +673,10 @@ TEST_F(InductionVarRangeTest, SymbolicTripCountDown) { ASSERT_TRUE(lower->InputAt(1)->IsIntConstant()); EXPECT_EQ(1, lower->InputAt(1)->AsIntConstant()->GetValue()); lower = lower->InputAt(0); - ASSERT_TRUE(lower->IsNeg()); - lower = lower->InputAt(0); ASSERT_TRUE(lower->IsSub()); - EXPECT_TRUE(lower->InputAt(0)->IsParameterValue()); - ASSERT_TRUE(lower->InputAt(1)->IsIntConstant()); - EXPECT_EQ(1000, lower->InputAt(1)->AsIntConstant()->GetValue()); + ASSERT_TRUE(lower->InputAt(0)->IsIntConstant()); + EXPECT_EQ(1000, lower->InputAt(0)->AsIntConstant()->GetValue()); + EXPECT_TRUE(lower->InputAt(1)->IsParameterValue()); // Verify upper is 1000-0. ASSERT_TRUE(upper != nullptr); @@ -661,7 +687,7 @@ TEST_F(InductionVarRangeTest, SymbolicTripCountDown) { EXPECT_EQ(0, upper->InputAt(1)->AsIntConstant()->GetValue()); // Verify taken-test is 1000>V. - range.GenerateTakenTest(increment_, graph_, loop_preheader_, &taken); + range_.GenerateTakenTest(increment_, graph_, loop_preheader_, &taken); ASSERT_TRUE(taken != nullptr); ASSERT_TRUE(taken->IsGreaterThan()); ASSERT_TRUE(taken->InputAt(0)->IsIntConstant()); diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index 6946265abe..67097deaeb 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -211,7 +211,7 @@ bool InstructionSimplifierVisitor::ReplaceRotateWithRor(HBinaryOperation* op, // Try to replace a binary operation flanked by one UShr and one Shl with a bitfield rotation. bool InstructionSimplifierVisitor::TryReplaceWithRotate(HBinaryOperation* op) { - // This simplification is currently supported on ARM and ARM64. + // This simplification is currently supported on x86, x86_64, ARM and ARM64. // TODO: Implement it for MIPS/64. const InstructionSet instruction_set = GetGraph()->GetInstructionSet(); switch (instruction_set) { @@ -778,9 +778,9 @@ void InstructionSimplifierVisitor::VisitCondition(HCondition* condition) { // Try to fold an HCompare into this HCondition. // This simplification is currently supported on x86, x86_64, ARM and ARM64. - // TODO: Implement it for MIPS and MIPS64. + // TODO: Implement it for MIPS64. InstructionSet instruction_set = GetGraph()->GetInstructionSet(); - if (instruction_set == kMips || instruction_set == kMips64) { + if (instruction_set == kMips64) { return; } @@ -1268,7 +1268,7 @@ void InstructionSimplifierVisitor::SimplifyStringEquals(HInvoke* instruction) { void InstructionSimplifierVisitor::SimplifyRotate(HInvoke* invoke, bool is_left) { DCHECK(invoke->IsInvokeStaticOrDirect()); DCHECK_EQ(invoke->GetOriginalInvokeType(), InvokeType::kStatic); - // This simplification is currently supported on ARM and ARM64. + // This simplification is currently supported on x86, x86_64, ARM and ARM64. // TODO: Implement it for MIPS/64. const InstructionSet instruction_set = GetGraph()->GetInstructionSet(); switch (instruction_set) { diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index 422bb8928d..fd454d8322 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -1790,12 +1790,27 @@ static void GenUnsafeGet(HInvoke* invoke, Location output_loc = locations->Out(); switch (type) { - case Primitive::kPrimInt: - case Primitive::kPrimNot: { + case Primitive::kPrimInt: { Register output = output_loc.AsRegister<Register>(); __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); - if (type == Primitive::kPrimNot) { - codegen->MaybeGenerateReadBarrier(invoke, output_loc, output_loc, base_loc, 0U, offset_loc); + break; + } + + case Primitive::kPrimNot: { + Register output = output_loc.AsRegister<Register>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + codegen->GenerateArrayLoadWithBakerReadBarrier( + invoke, output_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false); + } else { + __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); + codegen->GenerateReadBarrierSlow( + invoke, output_loc, output_loc, base_loc, 0U, offset_loc); + } + } else { + __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); + __ MaybeUnpoisonHeapReference(output); } break; } @@ -1823,8 +1838,10 @@ static void GenUnsafeGet(HInvoke* invoke, } } -static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke, - bool is_long, bool is_volatile) { +static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, + HInvoke* invoke, + Primitive::Type type, + bool is_volatile) { bool can_call = kEmitCompilerReadBarrier && (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); @@ -1836,7 +1853,7 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); - if (is_long) { + if (type == Primitive::kPrimLong) { if (is_volatile) { // Need to use XMM to read volatile. locations->AddTemp(Location::RequiresFpuRegister()); @@ -1847,25 +1864,30 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke } else { locations->SetOut(Location::RequiresRegister()); } + if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in InstructionCodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); + } } void IntrinsicLocationsBuilderX86::VisitUnsafeGet(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, /* is_long */ false, /* is_volatile */ false); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt, /* is_volatile */ false); } void IntrinsicLocationsBuilderX86::VisitUnsafeGetVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, /* is_long */ false, /* is_volatile */ true); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt, /* is_volatile */ true); } void IntrinsicLocationsBuilderX86::VisitUnsafeGetLong(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, /* is_long */ true, /* is_volatile */ false); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong, /* is_volatile */ false); } void IntrinsicLocationsBuilderX86::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, /* is_long */ true, /* is_volatile */ true); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong, /* is_volatile */ true); } void IntrinsicLocationsBuilderX86::VisitUnsafeGetObject(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, /* is_long */ false, /* is_volatile */ false); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot, /* is_volatile */ false); } void IntrinsicLocationsBuilderX86::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, /* is_long */ false, /* is_volatile */ true); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot, /* is_volatile */ true); } diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index ac9b24503c..ce737e3f7e 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -1917,16 +1917,30 @@ static void GenUnsafeGet(HInvoke* invoke, Location offset_loc = locations->InAt(2); CpuRegister offset = offset_loc.AsRegister<CpuRegister>(); Location output_loc = locations->Out(); - CpuRegister output = locations->Out().AsRegister<CpuRegister>(); + CpuRegister output = output_loc.AsRegister<CpuRegister>(); switch (type) { case Primitive::kPrimInt: - case Primitive::kPrimNot: __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); - if (type == Primitive::kPrimNot) { - codegen->MaybeGenerateReadBarrier(invoke, output_loc, output_loc, base_loc, 0U, offset_loc); + break; + + case Primitive::kPrimNot: { + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + codegen->GenerateArrayLoadWithBakerReadBarrier( + invoke, output_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false); + } else { + __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); + codegen->GenerateReadBarrierSlow( + invoke, output_loc, output_loc, base_loc, 0U, offset_loc); + } + } else { + __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); + __ MaybeUnpoisonHeapReference(output); } break; + } case Primitive::kPrimLong: __ movq(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); @@ -1938,7 +1952,9 @@ static void GenUnsafeGet(HInvoke* invoke, } } -static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { +static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, + HInvoke* invoke, + Primitive::Type type) { bool can_call = kEmitCompilerReadBarrier && (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); @@ -1951,25 +1967,30 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); locations->SetOut(Location::RequiresRegister()); + if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in InstructionCodeGeneratorX86_64::GenerateArrayLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); + } } void IntrinsicLocationsBuilderX86_64::VisitUnsafeGet(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); } void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); } void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLong(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); } void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); } void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObject(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); } void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); } diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc index a385448104..b383f1e1ad 100644 --- a/compiler/optimizing/pc_relative_fixups_x86.cc +++ b/compiler/optimizing/pc_relative_fixups_x86.cc @@ -15,7 +15,6 @@ */ #include "pc_relative_fixups_x86.h" -#include "code_generator_x86.h" namespace art { namespace x86 { @@ -80,10 +79,6 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { } void VisitPackedSwitch(HPackedSwitch* switch_insn) OVERRIDE { - if (switch_insn->GetNumEntries() <= - InstructionCodeGeneratorX86::kPackedSwitchJumpTableThreshold) { - return; - } // We need to replace the HPackedSwitch with a HX86PackedSwitch in order to // address the constant area. InitializePCRelativeBasePointer(); |