diff options
Diffstat (limited to 'compiler/optimizing')
44 files changed, 4134 insertions, 703 deletions
diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc index 5bcc65b03b..e4dee46c39 100644 --- a/compiler/optimizing/builder.cc +++ b/compiler/optimizing/builder.cc @@ -171,6 +171,7 @@ HGraph* HGraphBuilder::BuildGraph(const DexFile::CodeItem& code_item) { const uint16_t* code_ptr = code_item.insns_; const uint16_t* code_end = code_item.insns_ + code_item.insns_size_in_code_units_; + code_start_ = code_ptr; // Setup the graph with the entry block and exit block. graph_ = new (arena_) HGraph(arena_); @@ -267,6 +268,13 @@ HBasicBlock* HGraphBuilder::FindBlockStartingAt(int32_t index) const { } template<typename T> +void HGraphBuilder::Unop_12x(const Instruction& instruction, Primitive::Type type) { + HInstruction* first = LoadLocal(instruction.VRegB(), type); + current_block_->AddInstruction(new (arena_) T(type, first)); + UpdateLocal(instruction.VRegA(), current_block_->GetLastInstruction()); +} + +template<typename T> void HGraphBuilder::Binop_23x(const Instruction& instruction, Primitive::Type type) { HInstruction* first = LoadLocal(instruction.VRegB(), type); HInstruction* second = LoadLocal(instruction.VRegC(), type); @@ -409,6 +417,7 @@ bool HGraphBuilder::BuildInvoke(const Instruction& instruction, DCHECK_EQ(argument_index, number_of_arguments); current_block_->AddInstruction(invoke); + latest_result_ = invoke; return true; } @@ -496,6 +505,62 @@ void HGraphBuilder::BuildArrayAccess(const Instruction& instruction, } } +void HGraphBuilder::BuildFilledNewArray(uint32_t dex_offset, + uint32_t type_index, + uint32_t number_of_vreg_arguments, + bool is_range, + uint32_t* args, + uint32_t register_index) { + HInstruction* length = GetIntConstant(number_of_vreg_arguments); + HInstruction* object = new (arena_) HNewArray(length, dex_offset, type_index); + current_block_->AddInstruction(object); + + const char* descriptor = dex_file_->StringByTypeIdx(type_index); + DCHECK_EQ(descriptor[0], '[') << descriptor; + char primitive = descriptor[1]; + DCHECK(primitive == 'I' + || primitive == 'L' + || primitive == '[') << descriptor; + bool is_reference_array = (primitive == 'L') || (primitive == '['); + Primitive::Type type = is_reference_array ? Primitive::kPrimNot : Primitive::kPrimInt; + + Temporaries temps(graph_, 1); + temps.Add(object); + for (size_t i = 0; i < number_of_vreg_arguments; ++i) { + HInstruction* value = LoadLocal(is_range ? register_index + i : args[i], type); + HInstruction* index = GetIntConstant(i); + current_block_->AddInstruction( + new (arena_) HArraySet(object, index, value, type, dex_offset)); + } + latest_result_ = object; +} + +template <typename T> +void HGraphBuilder::BuildFillArrayData(HInstruction* object, + const T* data, + uint32_t element_count, + Primitive::Type anticipated_type, + uint32_t dex_offset) { + for (uint32_t i = 0; i < element_count; ++i) { + HInstruction* index = GetIntConstant(i); + HInstruction* value = GetIntConstant(data[i]); + current_block_->AddInstruction(new (arena_) HArraySet( + object, index, value, anticipated_type, dex_offset)); + } +} + +void HGraphBuilder::BuildFillWideArrayData(HInstruction* object, + const int64_t* data, + uint32_t element_count, + uint32_t dex_offset) { + for (uint32_t i = 0; i < element_count; ++i) { + HInstruction* index = GetIntConstant(i); + HInstruction* value = GetLongConstant(data[i]); + current_block_->AddInstruction(new (arena_) HArraySet( + object, index, value, Primitive::kPrimLong, dex_offset)); + } +} + void HGraphBuilder::PotentiallyAddSuspendCheck(int32_t target_offset, uint32_t dex_offset) { if (target_offset <= 0) { // Unconditionnally add a suspend check to backward branches. We can remove @@ -678,6 +743,21 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 break; } + case Instruction::NEG_INT: { + Unop_12x<HNeg>(instruction, Primitive::kPrimInt); + break; + } + + case Instruction::NEG_LONG: { + Unop_12x<HNeg>(instruction, Primitive::kPrimLong); + break; + } + + case Instruction::NOT_INT: { + Unop_12x<HNot>(instruction, Primitive::kPrimInt); + break; + } + case Instruction::ADD_INT: { Binop_23x<HAdd>(instruction, Primitive::kPrimInt); break; @@ -708,11 +788,41 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 break; } + case Instruction::SUB_FLOAT: { + Binop_23x<HSub>(instruction, Primitive::kPrimFloat); + break; + } + + case Instruction::SUB_DOUBLE: { + Binop_23x<HSub>(instruction, Primitive::kPrimDouble); + break; + } + case Instruction::ADD_INT_2ADDR: { Binop_12x<HAdd>(instruction, Primitive::kPrimInt); break; } + case Instruction::MUL_INT: { + Binop_23x<HMul>(instruction, Primitive::kPrimInt); + break; + } + + case Instruction::MUL_LONG: { + Binop_23x<HMul>(instruction, Primitive::kPrimLong); + break; + } + + case Instruction::MUL_FLOAT: { + Binop_23x<HMul>(instruction, Primitive::kPrimFloat); + break; + } + + case Instruction::MUL_DOUBLE: { + Binop_23x<HMul>(instruction, Primitive::kPrimDouble); + break; + } + case Instruction::ADD_LONG_2ADDR: { Binop_12x<HAdd>(instruction, Primitive::kPrimLong); break; @@ -738,6 +848,36 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 break; } + case Instruction::SUB_FLOAT_2ADDR: { + Binop_12x<HSub>(instruction, Primitive::kPrimFloat); + break; + } + + case Instruction::SUB_DOUBLE_2ADDR: { + Binop_12x<HSub>(instruction, Primitive::kPrimDouble); + break; + } + + case Instruction::MUL_INT_2ADDR: { + Binop_12x<HMul>(instruction, Primitive::kPrimInt); + break; + } + + case Instruction::MUL_LONG_2ADDR: { + Binop_12x<HMul>(instruction, Primitive::kPrimLong); + break; + } + + case Instruction::MUL_FLOAT_2ADDR: { + Binop_12x<HMul>(instruction, Primitive::kPrimFloat); + break; + } + + case Instruction::MUL_DOUBLE_2ADDR: { + Binop_12x<HMul>(instruction, Primitive::kPrimDouble); + break; + } + case Instruction::ADD_INT_LIT16: { Binop_22s<HAdd>(instruction, false); break; @@ -748,6 +888,11 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 break; } + case Instruction::MUL_INT_LIT16: { + Binop_22s<HMul>(instruction, false); + break; + } + case Instruction::ADD_INT_LIT8: { Binop_22b<HAdd>(instruction, false); break; @@ -758,6 +903,11 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 break; } + case Instruction::MUL_INT_LIT8: { + Binop_22b<HMul>(instruction, false); + break; + } + case Instruction::NEW_INSTANCE: { current_block_->AddInstruction( new (arena_) HNewInstance(dex_offset, instruction.VRegB_21c())); @@ -765,10 +915,92 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 break; } + case Instruction::NEW_ARRAY: { + HInstruction* length = LoadLocal(instruction.VRegB_22c(), Primitive::kPrimInt); + current_block_->AddInstruction( + new (arena_) HNewArray(length, dex_offset, instruction.VRegC_22c())); + UpdateLocal(instruction.VRegA_22c(), current_block_->GetLastInstruction()); + break; + } + + case Instruction::FILLED_NEW_ARRAY: { + uint32_t number_of_vreg_arguments = instruction.VRegA_35c(); + uint32_t type_index = instruction.VRegB_35c(); + uint32_t args[5]; + instruction.GetVarArgs(args); + BuildFilledNewArray(dex_offset, type_index, number_of_vreg_arguments, false, args, 0); + break; + } + + case Instruction::FILLED_NEW_ARRAY_RANGE: { + uint32_t number_of_vreg_arguments = instruction.VRegA_3rc(); + uint32_t type_index = instruction.VRegB_3rc(); + uint32_t register_index = instruction.VRegC_3rc(); + BuildFilledNewArray( + dex_offset, type_index, number_of_vreg_arguments, true, nullptr, register_index); + break; + } + + case Instruction::FILL_ARRAY_DATA: { + Temporaries temps(graph_, 1); + HInstruction* array = LoadLocal(instruction.VRegA_31t(), Primitive::kPrimNot); + HNullCheck* null_check = new (arena_) HNullCheck(array, dex_offset); + current_block_->AddInstruction(null_check); + temps.Add(null_check); + + HInstruction* length = new (arena_) HArrayLength(null_check); + current_block_->AddInstruction(length); + + int32_t payload_offset = instruction.VRegB_31t() + dex_offset; + const Instruction::ArrayDataPayload* payload = + reinterpret_cast<const Instruction::ArrayDataPayload*>(code_start_ + payload_offset); + const uint8_t* data = payload->data; + uint32_t element_count = payload->element_count; + + // Implementation of this DEX instruction seems to be that the bounds check is + // done before doing any stores. + HInstruction* last_index = GetIntConstant(payload->element_count - 1); + current_block_->AddInstruction(new (arena_) HBoundsCheck(last_index, length, dex_offset)); + + switch (payload->element_width) { + case 1: + BuildFillArrayData(null_check, + reinterpret_cast<const int8_t*>(data), + element_count, + Primitive::kPrimByte, + dex_offset); + break; + case 2: + BuildFillArrayData(null_check, + reinterpret_cast<const int16_t*>(data), + element_count, + Primitive::kPrimShort, + dex_offset); + break; + case 4: + BuildFillArrayData(null_check, + reinterpret_cast<const int32_t*>(data), + element_count, + Primitive::kPrimInt, + dex_offset); + break; + case 8: + BuildFillWideArrayData(null_check, + reinterpret_cast<const int64_t*>(data), + element_count, + dex_offset); + break; + default: + LOG(FATAL) << "Unknown element width for " << payload->element_width; + } + break; + } + case Instruction::MOVE_RESULT: case Instruction::MOVE_RESULT_WIDE: case Instruction::MOVE_RESULT_OBJECT: - UpdateLocal(instruction.VRegA(), current_block_->GetLastInstruction()); + UpdateLocal(instruction.VRegA(), latest_result_); + latest_result_ = nullptr; break; case Instruction::CMP_LONG: { diff --git a/compiler/optimizing/builder.h b/compiler/optimizing/builder.h index e68cdb0b1d..b55ef0764a 100644 --- a/compiler/optimizing/builder.h +++ b/compiler/optimizing/builder.h @@ -48,7 +48,9 @@ class HGraphBuilder : public ValueObject { dex_file_(dex_file), dex_compilation_unit_(dex_compilation_unit), compiler_driver_(driver), - return_type_(Primitive::GetType(dex_compilation_unit_->GetShorty()[0])) {} + return_type_(Primitive::GetType(dex_compilation_unit_->GetShorty()[0])), + code_start_(nullptr), + latest_result_(nullptr) {} // Only for unit testing. HGraphBuilder(ArenaAllocator* arena, Primitive::Type return_type = Primitive::kPrimInt) @@ -64,7 +66,9 @@ class HGraphBuilder : public ValueObject { dex_file_(nullptr), dex_compilation_unit_(nullptr), compiler_driver_(nullptr), - return_type_(return_type) {} + return_type_(return_type), + code_start_(nullptr), + latest_result_(nullptr) {} HGraph* BuildGraph(const DexFile::CodeItem& code); @@ -95,6 +99,9 @@ class HGraphBuilder : public ValueObject { bool InitializeParameters(uint16_t number_of_parameters); template<typename T> + void Unop_12x(const Instruction& instruction, Primitive::Type type); + + template<typename T> void Binop_23x(const Instruction& instruction, Primitive::Type type); template<typename T> @@ -126,6 +133,31 @@ class HGraphBuilder : public ValueObject { uint32_t* args, uint32_t register_index); + // Builds a new array node and the instructions that fill it. + void BuildFilledNewArray(uint32_t dex_offset, + uint32_t type_index, + uint32_t number_of_vreg_arguments, + bool is_range, + uint32_t* args, + uint32_t register_index); + + // Fills the given object with data as specified in the fill-array-data + // instruction. Currently only used for non-reference and non-floating point + // arrays. + template <typename T> + void BuildFillArrayData(HInstruction* object, + const T* data, + uint32_t element_count, + Primitive::Type anticipated_type, + uint32_t dex_offset); + + // Fills the given object with data as specified in the fill-array-data + // instruction. The data must be for long and double arrays. + void BuildFillWideArrayData(HInstruction* object, + const int64_t* data, + uint32_t element_count, + uint32_t dex_offset); + ArenaAllocator* const arena_; // A list of the size of the dex code holding block information for @@ -148,6 +180,14 @@ class HGraphBuilder : public ValueObject { CompilerDriver* const compiler_driver_; const Primitive::Type return_type_; + // The pointer in the dex file where the instructions of the code item + // being currently compiled start. + const uint16_t* code_start_; + + // The last invoke or fill-new-array being built. Only to be + // used by move-result instructions. + HInstruction* latest_result_; + DISALLOW_COPY_AND_ASSIGN(HGraphBuilder); }; diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index 29dbd8b33d..c4286a401b 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -17,6 +17,7 @@ #include "code_generator.h" #include "code_generator_arm.h" +#include "code_generator_arm64.h" #include "code_generator_x86.h" #include "code_generator_x86_64.h" #include "compiled_method.h" @@ -36,7 +37,7 @@ void CodeGenerator::CompileBaseline(CodeAllocator* allocator, bool is_leaf) { const GrowableArray<HBasicBlock*>& blocks = GetGraph()->GetBlocks(); DCHECK(blocks.Get(0) == GetGraph()->GetEntryBlock()); DCHECK(GoesToNextBlock(GetGraph()->GetEntryBlock(), blocks.Get(1))); - block_labels_.SetSize(blocks.Size()); + Initialize(); DCHECK_EQ(frame_size_, kUninitializedFrameSize); if (!is_leaf) { @@ -54,7 +55,7 @@ void CodeGenerator::CompileBaseline(CodeAllocator* allocator, bool is_leaf) { HGraphVisitor* instruction_visitor = GetInstructionVisitor(); for (size_t i = 0, e = blocks.Size(); i < e; ++i) { HBasicBlock* block = blocks.Get(i); - Bind(GetLabelOf(block)); + Bind(block); for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { HInstruction* current = it.Current(); current->Accept(location_builder); @@ -76,13 +77,13 @@ void CodeGenerator::CompileOptimized(CodeAllocator* allocator) { const GrowableArray<HBasicBlock*>& blocks = GetGraph()->GetBlocks(); DCHECK(blocks.Get(0) == GetGraph()->GetEntryBlock()); DCHECK(GoesToNextBlock(GetGraph()->GetEntryBlock(), blocks.Get(1))); - block_labels_.SetSize(blocks.Size()); + Initialize(); GenerateFrameEntry(); HGraphVisitor* instruction_visitor = GetInstructionVisitor(); for (size_t i = 0, e = blocks.Size(); i < e; ++i) { HBasicBlock* block = blocks.Get(i); - Bind(GetLabelOf(block)); + Bind(block); for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { HInstruction* current = it.Current(); current->Accept(instruction_visitor); @@ -273,10 +274,6 @@ bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) con return current->GetBlockId() + 1 == next->GetBlockId(); } -Label* CodeGenerator::GetLabelOf(HBasicBlock* block) const { - return block_labels_.GetRawStorage() + block->GetBlockId(); -} - CodeGenerator* CodeGenerator::Create(ArenaAllocator* allocator, HGraph* graph, InstructionSet instruction_set) { @@ -285,6 +282,9 @@ CodeGenerator* CodeGenerator::Create(ArenaAllocator* allocator, case kThumb2: { return new (allocator) arm::CodeGeneratorARM(graph); } + case kArm64: { + return new (allocator) arm64::CodeGeneratorARM64(graph); + } case kMips: return nullptr; case kX86: { @@ -477,8 +477,7 @@ void CodeGenerator::RecordPcInfo(HInstruction* instruction, uint32_t dex_pc) { case Location::kRegister : { int id = location.reg(); stack_map_stream_.AddDexRegisterEntry(DexRegisterMap::kInRegister, id); - if (current->GetType() == Primitive::kPrimDouble - || current->GetType() == Primitive::kPrimLong) { + if (current->GetType() == Primitive::kPrimLong) { stack_map_stream_.AddDexRegisterEntry(DexRegisterMap::kInRegister, id); ++i; DCHECK_LT(i, environment_size); @@ -486,52 +485,55 @@ void CodeGenerator::RecordPcInfo(HInstruction* instruction, uint32_t dex_pc) { break; } + case Location::kFpuRegister : { + int id = location.reg(); + stack_map_stream_.AddDexRegisterEntry(DexRegisterMap::kInFpuRegister, id); + if (current->GetType() == Primitive::kPrimDouble) { + stack_map_stream_.AddDexRegisterEntry(DexRegisterMap::kInFpuRegister, id); + ++i; + DCHECK_LT(i, environment_size); + } + break; + } + default: LOG(FATAL) << "Unexpected kind " << location.GetKind(); } } } -size_t CodeGenerator::GetStackOffsetOfSavedRegister(size_t index) { - return first_register_slot_in_slow_path_ + index * GetWordSize(); -} - void CodeGenerator::SaveLiveRegisters(LocationSummary* locations) { RegisterSet* register_set = locations->GetLiveRegisters(); - uint32_t count = 0; + size_t stack_offset = first_register_slot_in_slow_path_; for (size_t i = 0, e = GetNumberOfCoreRegisters(); i < e; ++i) { if (register_set->ContainsCoreRegister(i)) { - size_t stack_offset = GetStackOffsetOfSavedRegister(count); - ++count; - SaveCoreRegister(Location::StackSlot(stack_offset), i); // If the register holds an object, update the stack mask. if (locations->RegisterContainsObject(i)) { locations->SetStackBit(stack_offset / kVRegSize); } + stack_offset += SaveCoreRegister(stack_offset, i); } } for (size_t i = 0, e = GetNumberOfFloatingPointRegisters(); i < e; ++i) { if (register_set->ContainsFloatingPointRegister(i)) { - LOG(FATAL) << "Unimplemented"; + stack_offset += SaveFloatingPointRegister(stack_offset, i); } } } void CodeGenerator::RestoreLiveRegisters(LocationSummary* locations) { RegisterSet* register_set = locations->GetLiveRegisters(); - uint32_t count = 0; + size_t stack_offset = first_register_slot_in_slow_path_; for (size_t i = 0, e = GetNumberOfCoreRegisters(); i < e; ++i) { if (register_set->ContainsCoreRegister(i)) { - size_t stack_offset = GetStackOffsetOfSavedRegister(count); - ++count; - RestoreCoreRegister(Location::StackSlot(stack_offset), i); + stack_offset += RestoreCoreRegister(stack_offset, i); } } for (size_t i = 0, e = GetNumberOfFloatingPointRegisters(); i < e; ++i) { if (register_set->ContainsFloatingPointRegister(i)) { - LOG(FATAL) << "Unimplemented"; + stack_offset += RestoreFloatingPointRegister(stack_offset, i); } } } diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index 4eba791723..220d745561 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -24,13 +24,13 @@ #include "memory_region.h" #include "nodes.h" #include "stack_map_stream.h" -#include "utils/assembler.h" namespace art { static size_t constexpr kVRegSize = 4; static size_t constexpr kUninitializedFrameSize = 0; +class Assembler; class CodeGenerator; class DexCompilationUnit; class SrcMap; @@ -53,18 +53,12 @@ struct PcInfo { class SlowPathCode : public ArenaObject { public: - SlowPathCode() : entry_label_(), exit_label_() {} + SlowPathCode() {} virtual ~SlowPathCode() {} - Label* GetEntryLabel() { return &entry_label_; } - Label* GetExitLabel() { return &exit_label_; } - virtual void EmitNativeCode(CodeGenerator* codegen) = 0; private: - Label entry_label_; - Label exit_label_; - DISALLOW_COPY_AND_ASSIGN(SlowPathCode); }; @@ -80,7 +74,6 @@ class CodeGenerator : public ArenaObject { HGraph* GetGraph() const { return graph_; } - Label* GetLabelOf(HBasicBlock* block) const; bool GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const; size_t GetStackSlotOfParameter(HParameterValue* parameter) const { @@ -90,9 +83,10 @@ class CodeGenerator : public ArenaObject { + parameter->GetIndex() * kVRegSize; } + virtual void Initialize() = 0; virtual void GenerateFrameEntry() = 0; virtual void GenerateFrameExit() = 0; - virtual void Bind(Label* label) = 0; + virtual void Bind(HBasicBlock* block) = 0; virtual void Move(HInstruction* instruction, Location location, HInstruction* move_for) = 0; virtual HGraphVisitor* GetLocationBuilder() = 0; virtual HGraphVisitor* GetInstructionVisitor() = 0; @@ -116,8 +110,18 @@ class CodeGenerator : public ArenaObject { virtual void DumpCoreRegister(std::ostream& stream, int reg) const = 0; virtual void DumpFloatingPointRegister(std::ostream& stream, int reg) const = 0; virtual InstructionSet GetInstructionSet() const = 0; - virtual void SaveCoreRegister(Location stack_location, uint32_t reg_id) = 0; - virtual void RestoreCoreRegister(Location stack_location, uint32_t reg_id) = 0; + // Saves the register in the stack. Returns the size taken on stack. + virtual size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) = 0; + // Restores the register from the stack. Returns the size taken on stack. + virtual size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) = 0; + virtual size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { + LOG(FATAL) << "Unimplemented"; + return 0u; + } + virtual size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { + LOG(FATAL) << "Unimplemented"; + return 0u; + } void RecordPcInfo(HInstruction* instruction, uint32_t dex_pc); @@ -151,6 +155,7 @@ class CodeGenerator : public ArenaObject { void ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const; bool* GetBlockedCoreRegisters() const { return blocked_core_registers_; } + bool* GetBlockedFloatingPointRegisters() const { return blocked_fpu_registers_; } protected: CodeGenerator(HGraph* graph, @@ -167,7 +172,6 @@ class CodeGenerator : public ArenaObject { number_of_fpu_registers_(number_of_fpu_registers), number_of_register_pairs_(number_of_register_pairs), graph_(graph), - block_labels_(graph->GetArena(), 0), pc_infos_(graph->GetArena(), 32), slow_paths_(graph->GetArena(), 8), is_leaf_(true), @@ -205,8 +209,6 @@ class CodeGenerator : public ArenaObject { HGraph* const graph_; - // Labels for each block that will be compiled. - GrowableArray<Label> block_labels_; GrowableArray<PcInfo> pc_infos_; GrowableArray<SlowPathCode*> slow_paths_; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 9be780216a..a3b31d89c9 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -60,7 +60,21 @@ class InvokeRuntimeCallingConvention : public CallingConvention<Register, DRegis #define __ reinterpret_cast<ArmAssembler*>(codegen->GetAssembler())-> -class NullCheckSlowPathARM : public SlowPathCode { +class SlowPathCodeARM : public SlowPathCode { + public: + SlowPathCodeARM() : entry_label_(), exit_label_() {} + + Label* GetEntryLabel() { return &entry_label_; } + Label* GetExitLabel() { return &exit_label_; } + + private: + Label entry_label_; + Label exit_label_; + + DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM); +}; + +class NullCheckSlowPathARM : public SlowPathCodeARM { public: explicit NullCheckSlowPathARM(HNullCheck* instruction) : instruction_(instruction) {} @@ -77,7 +91,7 @@ class NullCheckSlowPathARM : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); }; -class StackOverflowCheckSlowPathARM : public SlowPathCode { +class StackOverflowCheckSlowPathARM : public SlowPathCodeARM { public: StackOverflowCheckSlowPathARM() {} @@ -91,12 +105,13 @@ class StackOverflowCheckSlowPathARM : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathARM); }; -class SuspendCheckSlowPathARM : public SlowPathCode { +class SuspendCheckSlowPathARM : public SlowPathCodeARM { public: explicit SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) : instruction_(instruction), successor_(successor) {} virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); codegen->SaveLiveRegisters(instruction_->GetLocations()); int32_t offset = QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pTestSuspend).Int32Value(); @@ -107,7 +122,7 @@ class SuspendCheckSlowPathARM : public SlowPathCode { if (successor_ == nullptr) { __ b(GetReturnLabel()); } else { - __ b(codegen->GetLabelOf(successor_)); + __ b(arm_codegen->GetLabelOf(successor_)); } } @@ -127,7 +142,7 @@ class SuspendCheckSlowPathARM : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); }; -class BoundsCheckSlowPathARM : public SlowPathCode { +class BoundsCheckSlowPathARM : public SlowPathCodeARM { public: BoundsCheckSlowPathARM(HBoundsCheck* instruction, Location index_location, @@ -137,7 +152,7 @@ class BoundsCheckSlowPathARM : public SlowPathCode { length_location_(length_location) {} virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = reinterpret_cast<CodeGeneratorARM*>(codegen); + CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); InvokeRuntimeCallingConvention calling_convention; arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), index_location_); @@ -195,16 +210,19 @@ void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) stream << ArmManagedRegister::FromDRegister(DRegister(reg)); } -void CodeGeneratorARM::SaveCoreRegister(Location stack_location, uint32_t reg_id) { - __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_location.GetStackIndex()); +size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { + __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); + return kArmWordSize; } -void CodeGeneratorARM::RestoreCoreRegister(Location stack_location, uint32_t reg_id) { - __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_location.GetStackIndex()); +size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { + __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); + return kArmWordSize; } CodeGeneratorARM::CodeGeneratorARM(HGraph* graph) : CodeGenerator(graph, kNumberOfCoreRegisters, kNumberOfDRegisters, kNumberOfRegisterPairs), + block_labels_(graph->GetArena(), 0), location_builder_(graph, this), instruction_visitor_(graph, this), move_resolver_(graph->GetArena(), this), @@ -220,19 +238,12 @@ Location CodeGeneratorARM::AllocateFreeRegister(Primitive::Type type) const { size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs); ArmManagedRegister pair = ArmManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg)); + DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]); + DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]); + blocked_core_registers_[pair.AsRegisterPairLow()] = true; blocked_core_registers_[pair.AsRegisterPairHigh()] = true; - // Block all other register pairs that share a register with `pair`. - for (int i = 0; i < kNumberOfRegisterPairs; i++) { - ArmManagedRegister current = - ArmManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i)); - if (current.AsRegisterPairLow() == pair.AsRegisterPairLow() - || current.AsRegisterPairLow() == pair.AsRegisterPairHigh() - || current.AsRegisterPairHigh() == pair.AsRegisterPairLow() - || current.AsRegisterPairHigh() == pair.AsRegisterPairHigh()) { - blocked_register_pairs_[i] = true; - } - } + UpdateBlockedPairRegisters(); return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh()); } @@ -278,7 +289,6 @@ void CodeGeneratorARM::SetupBlockedRegisters() const { // Reserve R4 for suspend check. blocked_core_registers_[R4] = true; - blocked_register_pairs_[R4_R5] = true; // Reserve thread register. blocked_core_registers_[TR] = true; @@ -302,6 +312,19 @@ void CodeGeneratorARM::SetupBlockedRegisters() const { blocked_fpu_registers_[D13] = true; blocked_fpu_registers_[D14] = true; blocked_fpu_registers_[D15] = true; + + UpdateBlockedPairRegisters(); +} + +void CodeGeneratorARM::UpdateBlockedPairRegisters() const { + for (int i = 0; i < kNumberOfRegisterPairs; i++) { + ArmManagedRegister current = + ArmManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i)); + if (blocked_core_registers_[current.AsRegisterPairLow()] + || blocked_core_registers_[current.AsRegisterPairHigh()]) { + blocked_register_pairs_[i] = true; + } + } } InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) @@ -313,7 +336,7 @@ void CodeGeneratorARM::GenerateFrameEntry() { bool skip_overflow_check = IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); if (!skip_overflow_check) { if (kExplicitStackOverflowCheck) { - SlowPathCode* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathARM(); + SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathARM(); AddSlowPath(slow_path); __ LoadFromOffset(kLoadWord, IP, TR, Thread::StackEndOffset<kArmWordSize>().Int32Value()); @@ -339,8 +362,8 @@ void CodeGeneratorARM::GenerateFrameExit() { __ PopList(1 << PC | 1 << R6 | 1 << R7); } -void CodeGeneratorARM::Bind(Label* label) { - __ Bind(label); +void CodeGeneratorARM::Bind(HBasicBlock* block) { + __ Bind(GetLabelOf(block)); } Location CodeGeneratorARM::GetStackLocation(HLoadLocal* load) const { @@ -635,42 +658,57 @@ void LocationsBuilderARM::VisitIf(HIf* if_instr) { new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall); HInstruction* cond = if_instr->InputAt(0); if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) { - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::RequiresRegister()); } } void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { HInstruction* cond = if_instr->InputAt(0); - if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) { - // Condition has been materialized, compare the output to 0 - DCHECK(if_instr->GetLocations()->InAt(0).IsRegister()); - __ cmp(if_instr->GetLocations()->InAt(0).As<Register>(), - ShifterOperand(0)); - __ b(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()), NE); + if (cond->IsIntConstant()) { + // Constant condition, statically compared against 1. + int32_t cond_value = cond->AsIntConstant()->GetValue(); + if (cond_value == 1) { + if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), + if_instr->IfTrueSuccessor())) { + __ b(codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + } + return; + } else { + DCHECK_EQ(cond_value, 0); + } } else { - // Condition has not been materialized, use its inputs as the comparison and its - // condition as the branch condition. - LocationSummary* locations = cond->GetLocations(); - if (locations->InAt(1).IsRegister()) { - __ cmp(locations->InAt(0).As<Register>(), - ShifterOperand(locations->InAt(1).As<Register>())); + if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) { + // Condition has been materialized, compare the output to 0 + DCHECK(if_instr->GetLocations()->InAt(0).IsRegister()); + __ cmp(if_instr->GetLocations()->InAt(0).As<Register>(), + ShifterOperand(0)); + __ b(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()), NE); } else { - DCHECK(locations->InAt(1).IsConstant()); - int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); - ShifterOperand operand; - if (ShifterOperand::CanHoldArm(value, &operand)) { - __ cmp(locations->InAt(0).As<Register>(), ShifterOperand(value)); + // Condition has not been materialized, use its inputs as the + // comparison and its condition as the branch condition. + LocationSummary* locations = cond->GetLocations(); + if (locations->InAt(1).IsRegister()) { + __ cmp(locations->InAt(0).As<Register>(), + ShifterOperand(locations->InAt(1).As<Register>())); } else { - Register temp = IP; - __ LoadImmediate(temp, value); - __ cmp(locations->InAt(0).As<Register>(), ShifterOperand(temp)); + DCHECK(locations->InAt(1).IsConstant()); + int32_t value = + locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + ShifterOperand operand; + if (ShifterOperand::CanHoldArm(value, &operand)) { + __ cmp(locations->InAt(0).As<Register>(), ShifterOperand(value)); + } else { + Register temp = IP; + __ LoadImmediate(temp, value); + __ cmp(locations->InAt(0).As<Register>(), ShifterOperand(temp)); + } } + __ b(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()), + ARMCondition(cond->AsCondition()->GetCondition())); } - __ b(codegen_->GetLabelOf(if_instr->IfTrueSuccessor()), - ARMCondition(cond->AsCondition()->GetCondition())); } - - if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), if_instr->IfFalseSuccessor())) { + if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), + if_instr->IfFalseSuccessor())) { __ b(codegen_->GetLabelOf(if_instr->IfFalseSuccessor())); } } @@ -679,10 +717,10 @@ void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { void LocationsBuilderARM::VisitCondition(HCondition* comp) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt(1, Location::RegisterOrConstant(comp->InputAt(1)), Location::kDiesAtEntry); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(comp->InputAt(1))); if (comp->NeedsMaterialization()) { - locations->SetOut(Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } } @@ -810,6 +848,7 @@ void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { } void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant) { + // Will be generated at use site. } void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { @@ -822,6 +861,26 @@ void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant) { // Will be generated at use site. } +void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant) { + // Will be generated at use site. +} + +void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant) { + // Will be generated at use site. +} + void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { ret->SetLocations(nullptr); } @@ -979,16 +1038,79 @@ void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } +void LocationsBuilderARM::VisitNeg(HNeg* neg) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); + switch (neg->GetResultType()) { + case Primitive::kPrimInt: + case Primitive::kPrimLong: { + bool output_overlaps = (neg->GetResultType() == Primitive::kPrimLong); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), output_overlaps); + break; + } + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + break; + + default: + LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); + } +} + +void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { + LocationSummary* locations = neg->GetLocations(); + Location out = locations->Out(); + Location in = locations->InAt(0); + switch (neg->GetResultType()) { + case Primitive::kPrimInt: + DCHECK(in.IsRegister()); + __ rsb(out.As<Register>(), in.As<Register>(), ShifterOperand(0)); + break; + + case Primitive::kPrimLong: + DCHECK(in.IsRegisterPair()); + // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) + __ rsbs(out.AsRegisterPairLow<Register>(), + in.AsRegisterPairLow<Register>(), + ShifterOperand(0)); + // We cannot emit an RSC (Reverse Subtract with Carry) + // instruction here, as it does not exist in the Thumb-2 + // instruction set. We use the following approach + // using SBC and SUB instead. + // + // out.hi = -C + __ sbc(out.AsRegisterPairHigh<Register>(), + out.AsRegisterPairHigh<Register>(), + ShifterOperand(out.AsRegisterPairHigh<Register>())); + // out.hi = out.hi - in.hi + __ sub(out.AsRegisterPairHigh<Register>(), + out.AsRegisterPairHigh<Register>(), + ShifterOperand(in.AsRegisterPairHigh<Register>())); + break; + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + break; + + default: + LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); + } +} + void LocationsBuilderARM::VisitAdd(HAdd* add) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); switch (add->GetResultType()) { case Primitive::kPrimInt: case Primitive::kPrimLong: { - bool dies_at_entry = add->GetResultType() != Primitive::kPrimLong; - locations->SetInAt(0, Location::RequiresRegister(), dies_at_entry); - locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)), dies_at_entry); - locations->SetOut(Location::RequiresRegister()); + bool output_overlaps = (add->GetResultType() == Primitive::kPrimLong); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); + locations->SetOut(Location::RequiresRegister(), output_overlaps); break; } @@ -1051,59 +1173,149 @@ void LocationsBuilderARM::VisitSub(HSub* sub) { switch (sub->GetResultType()) { case Primitive::kPrimInt: case Primitive::kPrimLong: { - bool dies_at_entry = sub->GetResultType() != Primitive::kPrimLong; - locations->SetInAt(0, Location::RequiresRegister(), dies_at_entry); - locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1)), dies_at_entry); - locations->SetOut(Location::RequiresRegister()); + bool output_overlaps = (sub->GetResultType() == Primitive::kPrimLong); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); + locations->SetOut(Location::RequiresRegister(), output_overlaps); break; } - - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister()); break; - + } default: - LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType(); + LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); } } void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { LocationSummary* locations = sub->GetLocations(); + Location out = locations->Out(); + Location first = locations->InAt(0); + Location second = locations->InAt(1); switch (sub->GetResultType()) { case Primitive::kPrimInt: { - if (locations->InAt(1).IsRegister()) { - __ sub(locations->Out().As<Register>(), - locations->InAt(0).As<Register>(), - ShifterOperand(locations->InAt(1).As<Register>())); + if (second.IsRegister()) { + __ sub(out.As<Register>(), first.As<Register>(), ShifterOperand(second.As<Register>())); } else { - __ AddConstant(locations->Out().As<Register>(), - locations->InAt(0).As<Register>(), - -locations->InAt(1).GetConstant()->AsIntConstant()->GetValue()); + __ AddConstant(out.As<Register>(), + first.As<Register>(), + -second.GetConstant()->AsIntConstant()->GetValue()); } break; } - case Primitive::kPrimLong: - __ subs(locations->Out().AsRegisterPairLow<Register>(), - locations->InAt(0).AsRegisterPairLow<Register>(), - ShifterOperand(locations->InAt(1).AsRegisterPairLow<Register>())); - __ sbc(locations->Out().AsRegisterPairHigh<Register>(), - locations->InAt(0).AsRegisterPairHigh<Register>(), - ShifterOperand(locations->InAt(1).AsRegisterPairHigh<Register>())); + case Primitive::kPrimLong: { + __ subs(out.AsRegisterPairLow<Register>(), + first.AsRegisterPairLow<Register>(), + ShifterOperand(second.AsRegisterPairLow<Register>())); + __ sbc(out.AsRegisterPairHigh<Register>(), + first.AsRegisterPairHigh<Register>(), + ShifterOperand(second.AsRegisterPairHigh<Register>())); break; + } - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: + case Primitive::kPrimFloat: { + __ vsubs(FromDToLowS(out.As<DRegister>()), + FromDToLowS(first.As<DRegister>()), + FromDToLowS(second.As<DRegister>())); + break; + } + + case Primitive::kPrimDouble: { + __ vsubd(out.As<DRegister>(), first.As<DRegister>(), second.As<DRegister>()); + break; + } + + + default: LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); + } +} + +void LocationsBuilderARM::VisitMul(HMul* mul) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); + switch (mul->GetResultType()) { + case Primitive::kPrimInt: + case Primitive::kPrimLong: { + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + } + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + } + + default: + LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); + } +} + +void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { + LocationSummary* locations = mul->GetLocations(); + Location out = locations->Out(); + Location first = locations->InAt(0); + Location second = locations->InAt(1); + switch (mul->GetResultType()) { + case Primitive::kPrimInt: { + __ mul(out.As<Register>(), first.As<Register>(), second.As<Register>()); + break; + } + case Primitive::kPrimLong: { + Register out_hi = out.AsRegisterPairHigh<Register>(); + Register out_lo = out.AsRegisterPairLow<Register>(); + Register in1_hi = first.AsRegisterPairHigh<Register>(); + Register in1_lo = first.AsRegisterPairLow<Register>(); + Register in2_hi = second.AsRegisterPairHigh<Register>(); + Register in2_lo = second.AsRegisterPairLow<Register>(); + + // Extra checks to protect caused by the existence of R1_R2. + // The algorithm is wrong if out.hi is either in1.lo or in2.lo: + // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); + DCHECK_NE(out_hi, in1_lo); + DCHECK_NE(out_hi, in2_lo); + + // input: in1 - 64 bits, in2 - 64 bits + // output: out + // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo + // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] + // parts: out.lo = (in1.lo * in2.lo)[31:0] + + // IP <- in1.lo * in2.hi + __ mul(IP, in1_lo, in2_hi); + // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo + __ mla(out_hi, in1_hi, in2_lo, IP); + // out.lo <- (in1.lo * in2.lo)[31:0]; + __ umull(out_lo, IP, in1_lo, in2_lo); + // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] + __ add(out_hi, out_hi, ShifterOperand(IP)); + break; + } + + case Primitive::kPrimFloat: { + __ vmuls(FromDToLowS(out.As<DRegister>()), + FromDToLowS(first.As<DRegister>()), + FromDToLowS(second.As<DRegister>())); break; + } + + case Primitive::kPrimDouble: { + __ vmuld(out.As<DRegister>(), first.As<DRegister>(), second.As<DRegister>()); + break; + } default: - LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType(); + LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); } } @@ -1129,6 +1341,29 @@ void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { DCHECK(!codegen_->IsLeafMethod()); } +void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); + InvokeRuntimeCallingConvention calling_convention; + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetOut(Location::RegisterLocation(R0)); + locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); +} + +void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { + InvokeRuntimeCallingConvention calling_convention; + LoadCurrentMethod(calling_convention.GetRegisterAt(1)); + __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex()); + + int32_t offset = QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocArrayWithAccessCheck).Int32Value(); + __ LoadFromOffset(kLoadWord, LR, TR, offset); + __ blx(LR); + + codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + DCHECK(!codegen_->IsLeafMethod()); +} + void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); @@ -1145,25 +1380,41 @@ void InstructionCodeGeneratorARM::VisitParameterValue(HParameterValue* instructi // Nothing to do, the parameter is already at its location. } -void LocationsBuilderARM::VisitNot(HNot* instruction) { +void LocationsBuilderARM::VisitNot(HNot* not_) { LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } -void InstructionCodeGeneratorARM::VisitNot(HNot* instruction) { - LocationSummary* locations = instruction->GetLocations(); - __ eor(locations->Out().As<Register>(), - locations->InAt(0).As<Register>(), ShifterOperand(1)); +void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { + LocationSummary* locations = not_->GetLocations(); + Location out = locations->Out(); + Location in = locations->InAt(0); + switch (not_->InputAt(0)->GetType()) { + case Primitive::kPrimBoolean: + __ eor(out.As<Register>(), in.As<Register>(), ShifterOperand(1)); + break; + + case Primitive::kPrimInt: + __ mvn(out.As<Register>(), ShifterOperand(in.As<Register>())); + break; + + case Primitive::kPrimLong: + LOG(FATAL) << "Not yet implemented type for not operation " << not_->GetResultType(); + break; + + default: + LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); + } } void LocationsBuilderARM::VisitCompare(HCompare* compare) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt(1, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { @@ -1219,9 +1470,8 @@ void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); bool is_object_type = instruction->GetFieldType() == Primitive::kPrimNot; - bool dies_at_entry = !is_object_type; - locations->SetInAt(0, Location::RequiresRegister(), dies_at_entry); - locations->SetInAt(1, Location::RequiresRegister(), dies_at_entry); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); // Temporary registers for the write barrier. if (is_object_type) { locations->AddTemp(Location::RequiresRegister()); @@ -1281,8 +1531,8 @@ void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instr void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { @@ -1349,7 +1599,7 @@ void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { } void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { - SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); + SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); codegen_->AddSlowPath(slow_path); LocationSummary* locations = instruction->GetLocations(); @@ -1368,10 +1618,9 @@ void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt( - 1, Location::RegisterOrConstant(instruction->InputAt(1)), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { @@ -1481,10 +1730,9 @@ void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); } else { - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt( - 1, Location::RegisterOrConstant(instruction->InputAt(1)), Location::kDiesAtEntry); - locations->SetInAt(2, Location::RequiresRegister(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + locations->SetInAt(2, Location::RequiresRegister()); } } @@ -1571,8 +1819,8 @@ void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { @@ -1595,7 +1843,7 @@ void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { LocationSummary* locations = instruction->GetLocations(); - SlowPathCode* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM( + SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM( instruction, locations->InAt(0), locations->InAt(1)); codegen_->AddSlowPath(slow_path); diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index 874db0fd54..1fe8a7eacc 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -140,10 +140,10 @@ class CodeGeneratorARM : public CodeGenerator { virtual void GenerateFrameEntry() OVERRIDE; virtual void GenerateFrameExit() OVERRIDE; - virtual void Bind(Label* label) OVERRIDE; + virtual void Bind(HBasicBlock* block) OVERRIDE; virtual void Move(HInstruction* instruction, Location location, HInstruction* move_for) OVERRIDE; - virtual void SaveCoreRegister(Location stack_location, uint32_t reg_id) OVERRIDE; - virtual void RestoreCoreRegister(Location stack_location, uint32_t reg_id) OVERRIDE; + virtual size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + virtual size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; virtual size_t GetWordSize() const OVERRIDE { return kArmWordSize; @@ -164,6 +164,7 @@ class CodeGeneratorARM : public CodeGenerator { } virtual void SetupBlockedRegisters() const OVERRIDE; + virtual Location AllocateFreeRegister(Primitive::Type type) const OVERRIDE; virtual Location GetStackLocation(HLoadLocal* load) const OVERRIDE; @@ -171,6 +172,9 @@ class CodeGeneratorARM : public CodeGenerator { virtual void DumpCoreRegister(std::ostream& stream, int reg) const OVERRIDE; virtual void DumpFloatingPointRegister(std::ostream& stream, int reg) const OVERRIDE; + // Blocks all register pairs made out of blocked core registers. + void UpdateBlockedPairRegisters() const; + ParallelMoveResolverARM* GetMoveResolver() { return &move_resolver_; } @@ -187,7 +191,17 @@ class CodeGeneratorARM : public CodeGenerator { // Emit a write barrier. void MarkGCCard(Register temp, Register card, Register object, Register value); + Label* GetLabelOf(HBasicBlock* block) const { + return block_labels_.GetRawStorage() + block->GetBlockId(); + } + + virtual void Initialize() OVERRIDE { + block_labels_.SetSize(GetGraph()->GetBlocks().Size()); + } + private: + // Labels for each block that will be compiled. + GrowableArray<Label> block_labels_; LocationsBuilderARM location_builder_; InstructionCodeGeneratorARM instruction_visitor_; ParallelMoveResolverARM move_resolver_; diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc new file mode 100644 index 0000000000..79528ac128 --- /dev/null +++ b/compiler/optimizing/code_generator_arm64.cc @@ -0,0 +1,1205 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_arm64.h" + +#include "entrypoints/quick/quick_entrypoints.h" +#include "gc/accounting/card_table.h" +#include "mirror/array-inl.h" +#include "mirror/art_method.h" +#include "mirror/class.h" +#include "thread.h" +#include "utils/arm64/assembler_arm64.h" +#include "utils/assembler.h" +#include "utils/stack_checks.h" + + +using namespace vixl; // NOLINT(build/namespaces) + +#ifdef __ +#error "ARM64 Codegen VIXL macro-assembler macro already defined." +#endif + + +namespace art { + +namespace arm64 { + +static bool IsFPType(Primitive::Type type) { + return type == Primitive::kPrimFloat || type == Primitive::kPrimDouble; +} + +// TODO: clean-up some of the constant definitions. +static constexpr size_t kHeapRefSize = sizeof(mirror::HeapReference<mirror::Object>); +static constexpr int kCurrentMethodStackOffset = 0; + +namespace { +// Convenience helpers to ease conversion to and from VIXL operands. + +int VIXLRegCodeFromART(int code) { + // TODO: static check? + DCHECK_EQ(SP, 31); + DCHECK_EQ(WSP, 31); + DCHECK_EQ(XZR, 32); + DCHECK_EQ(WZR, 32); + if (code == SP) { + return vixl::kSPRegInternalCode; + } + if (code == XZR) { + return vixl::kZeroRegCode; + } + return code; +} + +int ARTRegCodeFromVIXL(int code) { + // TODO: static check? + DCHECK_EQ(SP, 31); + DCHECK_EQ(WSP, 31); + DCHECK_EQ(XZR, 32); + DCHECK_EQ(WZR, 32); + if (code == vixl::kSPRegInternalCode) { + return SP; + } + if (code == vixl::kZeroRegCode) { + return XZR; + } + return code; +} + +Register XRegisterFrom(Location location) { + return Register::XRegFromCode(VIXLRegCodeFromART(location.reg())); +} + +Register WRegisterFrom(Location location) { + return Register::WRegFromCode(VIXLRegCodeFromART(location.reg())); +} + +Register RegisterFrom(Location location, Primitive::Type type) { + DCHECK(type != Primitive::kPrimVoid && !IsFPType(type)); + return type == Primitive::kPrimLong ? XRegisterFrom(location) : WRegisterFrom(location); +} + +Register OutputRegister(HInstruction* instr) { + return RegisterFrom(instr->GetLocations()->Out(), instr->GetType()); +} + +Register InputRegisterAt(HInstruction* instr, int input_index) { + return RegisterFrom(instr->GetLocations()->InAt(input_index), + instr->InputAt(input_index)->GetType()); +} + +int64_t Int64ConstantFrom(Location location) { + HConstant* instr = location.GetConstant(); + return instr->IsIntConstant() ? instr->AsIntConstant()->GetValue() + : instr->AsLongConstant()->GetValue(); +} + +Operand OperandFrom(Location location, Primitive::Type type) { + if (location.IsRegister()) { + return Operand(RegisterFrom(location, type)); + } else { + return Operand(Int64ConstantFrom(location)); + } +} + +Operand InputOperandAt(HInstruction* instr, int input_index) { + return OperandFrom(instr->GetLocations()->InAt(input_index), + instr->InputAt(input_index)->GetType()); +} + +MemOperand StackOperandFrom(Location location) { + return MemOperand(sp, location.GetStackIndex()); +} + +MemOperand HeapOperand(const Register& base, Offset offset) { + // A heap reference must be 32bit, so fit in a W register. + DCHECK(base.IsW()); + return MemOperand(base.X(), offset.SizeValue()); +} + +MemOperand HeapOperandFrom(Location location, Primitive::Type type, Offset offset) { + return HeapOperand(RegisterFrom(location, type), offset); +} + +Location LocationFrom(const Register& reg) { + return Location::RegisterLocation(ARTRegCodeFromVIXL(reg.code())); +} + +} // namespace + +inline Condition ARM64Condition(IfCondition cond) { + switch (cond) { + case kCondEQ: return eq; + case kCondNE: return ne; + case kCondLT: return lt; + case kCondLE: return le; + case kCondGT: return gt; + case kCondGE: return ge; + default: + LOG(FATAL) << "Unknown if condition"; + } + return nv; // Unreachable. +} + +static const Register kRuntimeParameterCoreRegisters[] = { x0, x1, x2, x3, x4, x5, x6, x7 }; +static constexpr size_t kRuntimeParameterCoreRegistersLength = + arraysize(kRuntimeParameterCoreRegisters); +static const FPRegister kRuntimeParameterFpuRegisters[] = { }; +static constexpr size_t kRuntimeParameterFpuRegistersLength = 0; + +class InvokeRuntimeCallingConvention : public CallingConvention<Register, FPRegister> { + public: + static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); + + InvokeRuntimeCallingConvention() + : CallingConvention(kRuntimeParameterCoreRegisters, + kRuntimeParameterCoreRegistersLength, + kRuntimeParameterFpuRegisters, + kRuntimeParameterFpuRegistersLength) {} + + Location GetReturnLocation(Primitive::Type return_type); + + private: + DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention); +}; + +Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type return_type) { + DCHECK_NE(return_type, Primitive::kPrimVoid); + if (return_type == Primitive::kPrimFloat || return_type == Primitive::kPrimDouble) { + LOG(FATAL) << "Unimplemented return type " << return_type; + } + return LocationFrom(x0); +} + +#define __ reinterpret_cast<Arm64Assembler*>(codegen->GetAssembler())->vixl_masm_-> + +class SlowPathCodeARM64 : public SlowPathCode { + public: + SlowPathCodeARM64() : entry_label_(), exit_label_() {} + + vixl::Label* GetEntryLabel() { return &entry_label_; } + vixl::Label* GetExitLabel() { return &exit_label_; } + + private: + vixl::Label entry_label_; + vixl::Label exit_label_; + + DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64); +}; + +class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 { + public: + explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction, + Location index_location, + Location length_location) + : instruction_(instruction), + index_location_(index_location), + length_location_(length_location) {} + + virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorARM64* arm64_codegen = reinterpret_cast<CodeGeneratorARM64*>(codegen); + __ Bind(GetEntryLabel()); + InvokeRuntimeCallingConvention calling_convention; + arm64_codegen->MoveHelper(LocationFrom(calling_convention.GetRegisterAt(0)), + index_location_, Primitive::kPrimInt); + arm64_codegen->MoveHelper(LocationFrom(calling_convention.GetRegisterAt(1)), + length_location_, Primitive::kPrimInt); + size_t offset = QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pThrowArrayBounds).SizeValue(); + __ Ldr(lr, MemOperand(tr, offset)); + __ Blr(lr); + codegen->RecordPcInfo(instruction_, instruction_->GetDexPc()); + } + + private: + HBoundsCheck* const instruction_; + const Location index_location_; + const Location length_location_; + + DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64); +}; + +class NullCheckSlowPathARM64 : public SlowPathCodeARM64 { + public: + explicit NullCheckSlowPathARM64(HNullCheck* instr) : instruction_(instr) {} + + virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + __ Bind(GetEntryLabel()); + int32_t offset = QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pThrowNullPointer).Int32Value(); + __ Ldr(lr, MemOperand(tr, offset)); + __ Blr(lr); + codegen->RecordPcInfo(instruction_, instruction_->GetDexPc()); + } + + private: + HNullCheck* const instruction_; + + DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM64); +}; + +class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 { + public: + explicit SuspendCheckSlowPathARM64(HSuspendCheck* instruction, + HBasicBlock* successor) + : instruction_(instruction), successor_(successor) {} + + virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + size_t offset = QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pTestSuspend).SizeValue(); + __ Bind(GetEntryLabel()); + __ Ldr(lr, MemOperand(tr, offset)); + __ Blr(lr); + codegen->RecordPcInfo(instruction_, instruction_->GetDexPc()); + __ B(GetReturnLabel()); + } + + vixl::Label* GetReturnLabel() { + DCHECK(successor_ == nullptr); + return &return_label_; + } + + + private: + HSuspendCheck* const instruction_; + // If not null, the block to branch to after the suspend check. + HBasicBlock* const successor_; + + // If `successor_` is null, the label to branch to after the suspend check. + vixl::Label return_label_; + + DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64); +}; + +#undef __ + +Location InvokeDexCallingConventionVisitor::GetNextLocation(Primitive::Type type) { + Location next_location; + if (type == Primitive::kPrimVoid) { + LOG(FATAL) << "Unreachable type " << type; + } + + if (type == Primitive::kPrimFloat || type == Primitive::kPrimDouble) { + LOG(FATAL) << "Unimplemented type " << type; + } + + if (gp_index_ < calling_convention.GetNumberOfRegisters()) { + next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_)); + if (type == Primitive::kPrimLong) { + // Double stack slot reserved on the stack. + stack_index_++; + } + } else { // Stack. + if (type == Primitive::kPrimLong) { + next_location = Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_)); + // Double stack slot reserved on the stack. + stack_index_++; + } else { + next_location = Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_)); + } + } + // Move to the next register/stack slot. + gp_index_++; + stack_index_++; + return next_location; +} + +CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph) + : CodeGenerator(graph, + kNumberOfAllocatableRegisters, + kNumberOfAllocatableFloatingPointRegisters, + kNumberOfAllocatableRegisterPairs), + block_labels_(nullptr), + location_builder_(graph, this), + instruction_visitor_(graph, this) {} + +#define __ reinterpret_cast<Arm64Assembler*>(GetAssembler())->vixl_masm_-> + +void CodeGeneratorARM64::GenerateFrameEntry() { + // TODO: Add proper support for the stack overflow check. + UseScratchRegisterScope temps(assembler_.vixl_masm_); + Register temp = temps.AcquireX(); + __ Add(temp, sp, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64))); + __ Ldr(temp, MemOperand(temp, 0)); + RecordPcInfo(nullptr, 0); + + CPURegList preserved_regs = GetFramePreservedRegisters(); + int frame_size = GetFrameSize(); + core_spill_mask_ |= preserved_regs.list(); + + __ Str(w0, MemOperand(sp, -frame_size, PreIndex)); + __ PokeCPURegList(preserved_regs, frame_size - preserved_regs.TotalSizeInBytes()); + + // Stack layout: + // sp[frame_size - 8] : lr. + // ... : other preserved registers. + // sp[frame_size - regs_size]: first preserved register. + // ... : reserved frame space. + // sp[0] : context pointer. +} + +void CodeGeneratorARM64::GenerateFrameExit() { + int frame_size = GetFrameSize(); + CPURegList preserved_regs = GetFramePreservedRegisters(); + __ PeekCPURegList(preserved_regs, frame_size - preserved_regs.TotalSizeInBytes()); + __ Drop(frame_size); +} + +void CodeGeneratorARM64::Bind(HBasicBlock* block) { + __ Bind(GetLabelOf(block)); +} + +void CodeGeneratorARM64::MoveHelper(Location destination, + Location source, + Primitive::Type type) { + if (source.Equals(destination)) { + return; + } + if (destination.IsRegister()) { + Register dst = RegisterFrom(destination, type); + if (source.IsRegister()) { + Register src = RegisterFrom(source, type); + DCHECK(dst.IsSameSizeAndType(src)); + __ Mov(dst, src); + } else { + DCHECK(dst.Is64Bits() || !source.IsDoubleStackSlot()); + __ Ldr(dst, StackOperandFrom(source)); + } + } else { + DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot()); + if (source.IsRegister()) { + __ Str(RegisterFrom(source, type), StackOperandFrom(destination)); + } else { + UseScratchRegisterScope temps(assembler_.vixl_masm_); + Register temp = destination.IsDoubleStackSlot() ? temps.AcquireX() : temps.AcquireW(); + __ Ldr(temp, StackOperandFrom(source)); + __ Str(temp, StackOperandFrom(destination)); + } + } +} + +void CodeGeneratorARM64::Move(HInstruction* instruction, + Location location, + HInstruction* move_for) { + LocationSummary* locations = instruction->GetLocations(); + if (locations != nullptr && locations->Out().Equals(location)) { + return; + } + + Primitive::Type type = instruction->GetType(); + + if (instruction->IsIntConstant() || instruction->IsLongConstant()) { + int64_t value = instruction->IsIntConstant() ? instruction->AsIntConstant()->GetValue() + : instruction->AsLongConstant()->GetValue(); + if (location.IsRegister()) { + Register dst = RegisterFrom(location, type); + DCHECK((instruction->IsIntConstant() && dst.Is32Bits()) || + (instruction->IsLongConstant() && dst.Is64Bits())); + __ Mov(dst, value); + } else { + DCHECK(location.IsStackSlot() || location.IsDoubleStackSlot()); + UseScratchRegisterScope temps(assembler_.vixl_masm_); + Register temp = instruction->IsIntConstant() ? temps.AcquireW() : temps.AcquireX(); + __ Mov(temp, value); + __ Str(temp, StackOperandFrom(location)); + } + + } else if (instruction->IsLoadLocal()) { + uint32_t stack_slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal()); + switch (type) { + case Primitive::kPrimNot: + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + MoveHelper(location, Location::StackSlot(stack_slot), type); + break; + case Primitive::kPrimLong: + MoveHelper(location, Location::DoubleStackSlot(stack_slot), type); + break; + default: + LOG(FATAL) << "Unimplemented type" << type; + } + + } else { + DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary()); + MoveHelper(location, locations->Out(), type); + } +} + +size_t CodeGeneratorARM64::FrameEntrySpillSize() const { + return GetFramePreservedRegistersSize(); +} + +Location CodeGeneratorARM64::GetStackLocation(HLoadLocal* load) const { + Primitive::Type type = load->GetType(); + switch (type) { + case Primitive::kPrimNot: + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + return Location::StackSlot(GetStackSlot(load->GetLocal())); + case Primitive::kPrimLong: + return Location::DoubleStackSlot(GetStackSlot(load->GetLocal())); + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Unimplemented type " << type; + break; + case Primitive::kPrimVoid: + default: + LOG(FATAL) << "Unexpected type " << type; + } + LOG(FATAL) << "Unreachable"; + return Location::NoLocation(); +} + +void CodeGeneratorARM64::MarkGCCard(Register object, Register value) { + UseScratchRegisterScope temps(assembler_.vixl_masm_); + Register card = temps.AcquireX(); + Register temp = temps.AcquireX(); + vixl::Label done; + __ Cbz(value, &done); + __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64WordSize>().Int32Value())); + __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); + __ Strb(card, MemOperand(card, temp)); + __ Bind(&done); +} + +void CodeGeneratorARM64::SetupBlockedRegisters() const { + // Block reserved registers: + // ip0 (VIXL temporary) + // ip1 (VIXL temporary) + // xSuspend (Suspend counter) + // lr + // sp is not part of the allocatable registers, so we don't need to block it. + CPURegList reserved_core_registers = vixl_reserved_core_registers; + reserved_core_registers.Combine(runtime_reserved_core_registers); + // TODO: See if we should instead allow allocating but preserve those if used. + reserved_core_registers.Combine(quick_callee_saved_registers); + while (!reserved_core_registers.IsEmpty()) { + blocked_core_registers_[reserved_core_registers.PopLowestIndex().code()] = true; + } +} + +Location CodeGeneratorARM64::AllocateFreeRegister(Primitive::Type type) const { + if (type == Primitive::kPrimVoid) { + LOG(FATAL) << "Unreachable type " << type; + } + + if (type == Primitive::kPrimFloat || type == Primitive::kPrimDouble) { + LOG(FATAL) << "Unimplemented support for floating-point"; + } + + ssize_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfXRegisters); + DCHECK_NE(reg, -1); + blocked_core_registers_[reg] = true; + + if (IsFPType(type)) { + return Location::FpuRegisterLocation(reg); + } else { + return Location::RegisterLocation(reg); + } +} + +void CodeGeneratorARM64::DumpCoreRegister(std::ostream& stream, int reg) const { + stream << Arm64ManagedRegister::FromXRegister(XRegister(reg)); +} + +void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg) const { + stream << Arm64ManagedRegister::FromDRegister(DRegister(reg)); +} + +#undef __ +#define __ assembler_->vixl_masm_-> + +InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph, + CodeGeneratorARM64* codegen) + : HGraphVisitor(graph), + assembler_(codegen->GetAssembler()), + codegen_(codegen) {} + +#define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M) \ + M(ArrayGet) \ + M(ArraySet) \ + M(DoubleConstant) \ + M(FloatConstant) \ + M(Mul) \ + M(Neg) \ + M(NewArray) \ + M(ParallelMove) + +#define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode + +enum UnimplementedInstructionBreakCode { +#define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name), + FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION) +#undef ENUM_UNIMPLEMENTED_INSTRUCTION +}; + +#define DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS(name) \ + void InstructionCodeGeneratorARM64::Visit##name(H##name* instr) { \ + __ Brk(UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name)); \ + } \ + void LocationsBuilderARM64::Visit##name(H##name* instr) { \ + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); \ + locations->SetOut(Location::Any()); \ + } + FOR_EACH_UNIMPLEMENTED_INSTRUCTION(DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS) +#undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS + +#undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE + +void LocationsBuilderARM64::HandleAddSub(HBinaryOperation* instr) { + DCHECK(instr->IsAdd() || instr->IsSub()); + DCHECK_EQ(instr->InputCount(), 2U); + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); + Primitive::Type type = instr->GetResultType(); + switch (type) { + case Primitive::kPrimInt: + case Primitive::kPrimLong: { + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instr->InputAt(1))); + locations->SetOut(Location::RequiresRegister()); + break; + } + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + LOG(FATAL) << "Unexpected " << instr->DebugName() << " type " << type; + break; + default: + LOG(FATAL) << "Unimplemented " << instr->DebugName() << " type " << type; + } +} + +void InstructionCodeGeneratorARM64::HandleAddSub(HBinaryOperation* instr) { + DCHECK(instr->IsAdd() || instr->IsSub()); + + Primitive::Type type = instr->GetType(); + Register dst = OutputRegister(instr); + Register lhs = InputRegisterAt(instr, 0); + Operand rhs = InputOperandAt(instr, 1); + + switch (type) { + case Primitive::kPrimInt: + case Primitive::kPrimLong: + if (instr->IsAdd()) { + __ Add(dst, lhs, rhs); + } else { + __ Sub(dst, lhs, rhs); + } + break; + + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + LOG(FATAL) << "Unexpected add/sub type " << type; + break; + default: + LOG(FATAL) << "Unimplemented add/sub type " << type; + } +} + +void LocationsBuilderARM64::VisitAdd(HAdd* instruction) { + HandleAddSub(instruction); +} + +void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) { + HandleAddSub(instruction); +} + +void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) { + __ Ldr(OutputRegister(instruction), + HeapOperand(InputRegisterAt(instruction, 0), mirror::Array::LengthOffset())); +} + +void LocationsBuilderARM64::VisitCompare(HCompare* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + locations->SetOut(Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitCompare(HCompare* instruction) { + Primitive::Type in_type = instruction->InputAt(0)->GetType(); + + DCHECK_EQ(in_type, Primitive::kPrimLong); + switch (in_type) { + case Primitive::kPrimLong: { + vixl::Label done; + Register result = OutputRegister(instruction); + Register left = InputRegisterAt(instruction, 0); + Operand right = InputOperandAt(instruction, 1); + __ Subs(result, left, right); + __ B(eq, &done); + __ Mov(result, 1); + __ Cneg(result, result, le); + __ Bind(&done); + break; + } + default: + LOG(FATAL) << "Unimplemented compare type " << in_type; + } +} + +void LocationsBuilderARM64::VisitCondition(HCondition* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (instruction->NeedsMaterialization()) { + locations->SetOut(Location::RequiresRegister()); + } +} + +void InstructionCodeGeneratorARM64::VisitCondition(HCondition* instruction) { + if (!instruction->NeedsMaterialization()) { + return; + } + + LocationSummary* locations = instruction->GetLocations(); + Register lhs = InputRegisterAt(instruction, 0); + Operand rhs = InputOperandAt(instruction, 1); + Register res = RegisterFrom(locations->Out(), instruction->GetType()); + Condition cond = ARM64Condition(instruction->GetCondition()); + + __ Cmp(lhs, rhs); + __ Csel(res, vixl::Assembler::AppropriateZeroRegFor(res), Operand(1), InvertCondition(cond)); +} + +#define FOR_EACH_CONDITION_INSTRUCTION(M) \ + M(Equal) \ + M(NotEqual) \ + M(LessThan) \ + M(LessThanOrEqual) \ + M(GreaterThan) \ + M(GreaterThanOrEqual) +#define DEFINE_CONDITION_VISITORS(Name) \ +void LocationsBuilderARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); } \ +void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); } +FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS) +#undef FOR_EACH_CONDITION_INSTRUCTION + +void LocationsBuilderARM64::VisitExit(HExit* exit) { + exit->SetLocations(nullptr); +} + +void InstructionCodeGeneratorARM64::VisitExit(HExit* exit) { + if (kIsDebugBuild) { + down_cast<Arm64Assembler*>(GetAssembler())->Comment("Unreachable"); + __ Brk(0); // TODO: Introduce special markers for such code locations. + } +} + +void LocationsBuilderARM64::VisitGoto(HGoto* got) { + got->SetLocations(nullptr); +} + +void InstructionCodeGeneratorARM64::VisitGoto(HGoto* got) { + HBasicBlock* successor = got->GetSuccessor(); + // TODO: Support for suspend checks emission. + if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { + __ B(codegen_->GetLabelOf(successor)); + } +} + +void LocationsBuilderARM64::VisitIf(HIf* if_instr) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); + HInstruction* cond = if_instr->InputAt(0); + DCHECK(cond->IsCondition()); + if (cond->AsCondition()->NeedsMaterialization()) { + locations->SetInAt(0, Location::RequiresRegister()); + } +} + +void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) { + HInstruction* cond = if_instr->InputAt(0); + DCHECK(cond->IsCondition()); + HCondition* condition = cond->AsCondition(); + vixl::Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor()); + vixl::Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor()); + + // TODO: Support constant condition input in VisitIf. + + if (condition->NeedsMaterialization()) { + // The condition instruction has been materialized, compare the output to 0. + Location cond_val = if_instr->GetLocations()->InAt(0); + DCHECK(cond_val.IsRegister()); + __ Cbnz(InputRegisterAt(if_instr, 0), true_target); + + } else { + // The condition instruction has not been materialized, use its inputs as + // the comparison and its condition as the branch condition. + Register lhs = InputRegisterAt(condition, 0); + Operand rhs = InputOperandAt(condition, 1); + Condition cond = ARM64Condition(condition->GetCondition()); + if ((cond == eq || cond == ne) && rhs.IsImmediate() && (rhs.immediate() == 0)) { + if (cond == eq) { + __ Cbz(lhs, true_target); + } else { + __ Cbnz(lhs, true_target); + } + } else { + __ Cmp(lhs, rhs); + __ B(cond, true_target); + } + } + + if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), if_instr->IfFalseSuccessor())) { + __ B(false_target); + } +} + +void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { + Primitive::Type res_type = instruction->GetType(); + Register res = OutputRegister(instruction); + Register obj = InputRegisterAt(instruction, 0); + uint32_t offset = instruction->GetFieldOffset().Uint32Value(); + + switch (res_type) { + case Primitive::kPrimBoolean: { + __ Ldrb(res, MemOperand(obj, offset)); + break; + } + case Primitive::kPrimByte: { + __ Ldrsb(res, MemOperand(obj, offset)); + break; + } + case Primitive::kPrimShort: { + __ Ldrsh(res, MemOperand(obj, offset)); + break; + } + case Primitive::kPrimChar: { + __ Ldrh(res, MemOperand(obj, offset)); + break; + } + case Primitive::kPrimInt: + case Primitive::kPrimNot: + case Primitive::kPrimLong: { // TODO: support volatile. + DCHECK(res.IsX() == (res_type == Primitive::kPrimLong)); + __ Ldr(res, MemOperand(obj, offset)); + break; + } + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Unimplemented register res_type " << res_type; + break; + + case Primitive::kPrimVoid: + LOG(FATAL) << "Unreachable res_type " << res_type; + } +} + +void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { + Register obj = InputRegisterAt(instruction, 0); + Register value = InputRegisterAt(instruction, 1); + Primitive::Type field_type = instruction->InputAt(1)->GetType(); + uint32_t offset = instruction->GetFieldOffset().Uint32Value(); + + switch (field_type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: { + __ Strb(value, MemOperand(obj, offset)); + break; + } + + case Primitive::kPrimShort: + case Primitive::kPrimChar: { + __ Strh(value, MemOperand(obj, offset)); + break; + } + + case Primitive::kPrimInt: + case Primitive::kPrimNot: + case Primitive::kPrimLong: { + DCHECK(value.IsX() == (field_type == Primitive::kPrimLong)); + __ Str(value, MemOperand(obj, offset)); + + if (field_type == Primitive::kPrimNot) { + codegen_->MarkGCCard(obj, value); + } + break; + } + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Unimplemented register type " << field_type; + break; + + case Primitive::kPrimVoid: + LOG(FATAL) << "Unreachable type " << field_type; + } +} + +void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant) { + // Will be generated at use site. +} + +void LocationsBuilderARM64::VisitInvokeStatic(HInvokeStatic* invoke) { + HandleInvoke(invoke); +} + +void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { + HandleInvoke(invoke); +} + +void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall); + locations->AddTemp(LocationFrom(x0)); + + InvokeDexCallingConventionVisitor calling_convention_visitor; + for (size_t i = 0; i < invoke->InputCount(); i++) { + HInstruction* input = invoke->InputAt(i); + locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType())); + } + + Primitive::Type return_type = invoke->GetType(); + if (return_type != Primitive::kPrimVoid) { + locations->SetOut(calling_convention_visitor.GetReturnLocation(return_type)); + } +} + +void InstructionCodeGeneratorARM64::VisitInvokeStatic(HInvokeStatic* invoke) { + Register temp = WRegisterFrom(invoke->GetLocations()->GetTemp(0)); + // Make sure that ArtMethod* is passed in W0 as per the calling convention + DCHECK(temp.Is(w0)); + size_t index_in_cache = mirror::Array::DataOffset(kHeapRefSize).SizeValue() + + invoke->GetIndexInDexCache() * kHeapRefSize; + + // TODO: Implement all kinds of calls: + // 1) boot -> boot + // 2) app -> boot + // 3) app -> app + // + // Currently we implement the app -> app logic, which looks up in the resolve cache. + + // temp = method; + __ Ldr(temp, MemOperand(sp, kCurrentMethodStackOffset)); + // temp = temp->dex_cache_resolved_methods_; + __ Ldr(temp, MemOperand(temp.X(), mirror::ArtMethod::DexCacheResolvedMethodsOffset().SizeValue())); + // temp = temp[index_in_cache]; + __ Ldr(temp, MemOperand(temp.X(), index_in_cache)); + // lr = temp->entry_point_from_quick_compiled_code_; + __ Ldr(lr, MemOperand(temp.X(), mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset().SizeValue())); + // lr(); + __ Blr(lr); + + codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); + DCHECK(!codegen_->IsLeafMethod()); +} + +void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { + LocationSummary* locations = invoke->GetLocations(); + Location receiver = locations->InAt(0); + Register temp = XRegisterFrom(invoke->GetLocations()->GetTemp(0)); + size_t method_offset = mirror::Class::EmbeddedVTableOffset().SizeValue() + + invoke->GetVTableIndex() * sizeof(mirror::Class::VTableEntry); + Offset class_offset = mirror::Object::ClassOffset(); + Offset entry_point = mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(); + + // temp = object->GetClass(); + if (receiver.IsStackSlot()) { + __ Ldr(temp.W(), MemOperand(sp, receiver.GetStackIndex())); + __ Ldr(temp.W(), MemOperand(temp, class_offset.SizeValue())); + } else { + DCHECK(receiver.IsRegister()); + __ Ldr(temp.W(), HeapOperandFrom(receiver, Primitive::kPrimNot, + class_offset)); + } + // temp = temp->GetMethodAt(method_offset); + __ Ldr(temp.W(), MemOperand(temp, method_offset)); + // lr = temp->GetEntryPoint(); + __ Ldr(lr, MemOperand(temp, entry_point.SizeValue())); + // lr(); + __ Blr(lr); + DCHECK(!codegen_->IsLeafMethod()); + codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); +} + +void LocationsBuilderARM64::VisitLoadLocal(HLoadLocal* load) { + load->SetLocations(nullptr); +} + +void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load) { + // Nothing to do, this is driven by the code generator. +} + +void LocationsBuilderARM64::VisitLocal(HLocal* local) { + local->SetLocations(nullptr); +} + +void InstructionCodeGeneratorARM64::VisitLocal(HLocal* local) { + DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock()); +} + +void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant) { + // Will be generated at use site. +} + +void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); + InvokeRuntimeCallingConvention calling_convention; + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0))); + locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(1))); + locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); +} + +void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) { + LocationSummary* locations = instruction->GetLocations(); + Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt); + DCHECK(type_index.Is(w0)); + Register current_method = RegisterFrom(locations->GetTemp(1), Primitive::kPrimNot); + DCHECK(current_method.Is(w1)); + __ Ldr(current_method, MemOperand(sp, kCurrentMethodStackOffset)); + __ Mov(type_index, instruction->GetTypeIndex()); + __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocObjectWithAccessCheck).Int32Value())); + __ Blr(lr); + codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + DCHECK(!codegen_->IsLeafMethod()); +} + +void LocationsBuilderARM64::VisitNot(HNot* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister()); +} + +void InstructionCodeGeneratorARM64::VisitNot(HNot* instruction) { + switch (instruction->InputAt(0)->GetType()) { + case Primitive::kPrimBoolean: + __ Eor(OutputRegister(instruction), InputRegisterAt(instruction, 0), Operand(1)); + break; + + case Primitive::kPrimInt: + __ Mvn(OutputRegister(instruction), InputOperandAt(instruction, 0)); + break; + + case Primitive::kPrimLong: + LOG(FATAL) << "Not yet implemented type for not operation " << instruction->GetResultType(); + break; + + default: + LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType(); + } +} + +void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + locations->SetInAt(0, Location::RequiresRegister()); + if (instruction->HasUses()) { + locations->SetOut(Location::SameAsFirstInput()); + } +} + +void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) { + SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction); + codegen_->AddSlowPath(slow_path); + + LocationSummary* locations = instruction->GetLocations(); + Location obj = locations->InAt(0); + if (obj.IsRegister()) { + __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel()); + } else { + DCHECK(obj.IsConstant()) << obj; + DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0); + __ B(slow_path->GetEntryLabel()); + } +} + +void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); + if (location.IsStackSlot()) { + location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); + } else if (location.IsDoubleStackSlot()) { + location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); + } + locations->SetOut(location); +} + +void InstructionCodeGeneratorARM64::VisitParameterValue(HParameterValue* instruction) { + // Nothing to do, the parameter is already at its location. +} + +void LocationsBuilderARM64::VisitPhi(HPhi* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) { + locations->SetInAt(i, Location::Any()); + } + locations->SetOut(Location::Any()); +} + +void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction) { + LOG(FATAL) << "Unreachable"; +} + +void LocationsBuilderARM64::VisitReturn(HReturn* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + Primitive::Type return_type = instruction->InputAt(0)->GetType(); + + if (return_type == Primitive::kPrimFloat || return_type == Primitive::kPrimDouble) { + LOG(FATAL) << "Unimplemented return type " << return_type; + } + + locations->SetInAt(0, LocationFrom(x0)); +} + +void InstructionCodeGeneratorARM64::VisitReturn(HReturn* instruction) { + if (kIsDebugBuild) { + Primitive::Type type = instruction->InputAt(0)->GetType(); + switch (type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimNot: + DCHECK(InputRegisterAt(instruction, 0).Is(w0)); + break; + + case Primitive::kPrimLong: + DCHECK(InputRegisterAt(instruction, 0).Is(x0)); + break; + + default: + LOG(FATAL) << "Unimplemented return type " << type; + } + } + codegen_->GenerateFrameExit(); + __ Br(lr); +} + +void LocationsBuilderARM64::VisitReturnVoid(HReturnVoid* instruction) { + instruction->SetLocations(nullptr); +} + +void InstructionCodeGeneratorARM64::VisitReturnVoid(HReturnVoid* instruction) { + codegen_->GenerateFrameExit(); + __ Br(lr); +} + +void LocationsBuilderARM64::VisitStoreLocal(HStoreLocal* store) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(store); + Primitive::Type field_type = store->InputAt(1)->GetType(); + switch (field_type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimNot: + locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal()))); + break; + + case Primitive::kPrimLong: + locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal()))); + break; + + default: + LOG(FATAL) << "Unimplemented local type " << field_type; + } +} + +void InstructionCodeGeneratorARM64::VisitStoreLocal(HStoreLocal* store) { +} + +void LocationsBuilderARM64::VisitSub(HSub* instruction) { + HandleAddSub(instruction); +} + +void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) { + HandleAddSub(instruction); +} + +void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + if (instruction->HasUses()) { + locations->SetOut(Location::SameAsFirstInput()); + } +} + +void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) { + LocationSummary* locations = instruction->GetLocations(); + BoundsCheckSlowPathARM64* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64( + instruction, locations->InAt(0), locations->InAt(1)); + codegen_->AddSlowPath(slow_path); + + __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1)); + __ B(slow_path->GetEntryLabel(), hs); +} + +void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) { + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); +} + +void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction) { + // TODO: Improve support for suspend checks. + SuspendCheckSlowPathARM64* slow_path = + new (GetGraph()->GetArena()) SuspendCheckSlowPathARM64(instruction, nullptr); + codegen_->AddSlowPath(slow_path); + + __ Subs(wSuspend, wSuspend, 1); + __ B(slow_path->GetEntryLabel(), le); + __ Bind(slow_path->GetReturnLabel()); +} + +void LocationsBuilderARM64::VisitTemporary(HTemporary* temp) { + temp->SetLocations(nullptr); +} + +void InstructionCodeGeneratorARM64::VisitTemporary(HTemporary* temp) { + // Nothing to do, this is driven by the code generator. +} + +} // namespace arm64 +} // namespace art diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h new file mode 100644 index 0000000000..a4003ffea5 --- /dev/null +++ b/compiler/optimizing/code_generator_arm64.h @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ +#define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ + +#include "code_generator.h" +#include "nodes.h" +#include "parallel_move_resolver.h" +#include "utils/arm64/assembler_arm64.h" +#include "a64/disasm-a64.h" +#include "a64/macro-assembler-a64.h" +#include "arch/arm64/quick_method_frame_info_arm64.h" + +namespace art { +namespace arm64 { + +class CodeGeneratorARM64; + +static constexpr size_t kArm64WordSize = 8; +static const vixl::Register kParameterCoreRegisters[] = { + vixl::x1, vixl::x2, vixl::x3, vixl::x4, vixl::x5, vixl::x6, vixl::x7 +}; +static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); +static const vixl::FPRegister kParameterFPRegisters[] = { + vixl::d0, vixl::d1, vixl::d2, vixl::d3, vixl::d4, vixl::d5, vixl::d6, vixl::d7 +}; +static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters); + +const vixl::Register tr = vixl::x18; // Thread Register +const vixl::Register wSuspend = vixl::w19; // Suspend Register +const vixl::Register xSuspend = vixl::x19; + +const vixl::CPURegList vixl_reserved_core_registers(vixl::ip0, vixl::ip1); +const vixl::CPURegList runtime_reserved_core_registers(tr, xSuspend, vixl::lr); +const vixl::CPURegList quick_callee_saved_registers(vixl::CPURegister::kRegister, + vixl::kXRegSize, + kArm64CalleeSaveRefSpills); + +class InvokeDexCallingConvention : public CallingConvention<vixl::Register, vixl::FPRegister> { + public: + InvokeDexCallingConvention() + : CallingConvention(kParameterCoreRegisters, + kParameterCoreRegistersLength, + kParameterFPRegisters, + kParameterFPRegistersLength) {} + + Location GetReturnLocation(Primitive::Type return_type) { + DCHECK_NE(return_type, Primitive::kPrimVoid); + if (return_type == Primitive::kPrimFloat || return_type == Primitive::kPrimDouble) { + LOG(FATAL) << "Unimplemented return type " << return_type; + } + return Location::RegisterLocation(X0); + } + + + private: + DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention); +}; + +class InvokeDexCallingConventionVisitor { + public: + InvokeDexCallingConventionVisitor() : gp_index_(0), stack_index_(0) {} + + Location GetNextLocation(Primitive::Type type); + Location GetReturnLocation(Primitive::Type return_type) { + return calling_convention.GetReturnLocation(return_type); + } + + private: + InvokeDexCallingConvention calling_convention; + // The current index for core registers. + uint32_t gp_index_; + // The current stack index. + uint32_t stack_index_; + + DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitor); +}; + +class InstructionCodeGeneratorARM64 : public HGraphVisitor { + public: + InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen); + +#define DECLARE_VISIT_INSTRUCTION(name, super) \ + virtual void Visit##name(H##name* instr); + FOR_EACH_CONCRETE_INSTRUCTION(DECLARE_VISIT_INSTRUCTION) +#undef DECLARE_VISIT_INSTRUCTION + + void LoadCurrentMethod(XRegister reg); + + Arm64Assembler* GetAssembler() const { return assembler_; } + + private: + void HandleAddSub(HBinaryOperation* instr); + + Arm64Assembler* const assembler_; + CodeGeneratorARM64* const codegen_; + + DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64); +}; + +class LocationsBuilderARM64 : public HGraphVisitor { + public: + explicit LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen) + : HGraphVisitor(graph), codegen_(codegen) {} + +#define DECLARE_VISIT_INSTRUCTION(name, super) \ + virtual void Visit##name(H##name* instr); + FOR_EACH_CONCRETE_INSTRUCTION(DECLARE_VISIT_INSTRUCTION) +#undef DECLARE_VISIT_INSTRUCTION + + private: + void HandleAddSub(HBinaryOperation* instr); + void HandleInvoke(HInvoke* instr); + + CodeGeneratorARM64* const codegen_; + InvokeDexCallingConventionVisitor parameter_visitor_; + + DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64); +}; + +class CodeGeneratorARM64 : public CodeGenerator { + public: + explicit CodeGeneratorARM64(HGraph* graph); + virtual ~CodeGeneratorARM64() { } + + virtual void GenerateFrameEntry() OVERRIDE; + virtual void GenerateFrameExit() OVERRIDE; + + static const vixl::CPURegList& GetFramePreservedRegisters() { + static const vixl::CPURegList frame_preserved_regs = + vixl::CPURegList(vixl::CPURegister::kRegister, vixl::kXRegSize, vixl::lr.Bit()); + return frame_preserved_regs; + } + static int GetFramePreservedRegistersSize() { + return GetFramePreservedRegisters().TotalSizeInBytes(); + } + + virtual void Bind(HBasicBlock* block) OVERRIDE; + + vixl::Label* GetLabelOf(HBasicBlock* block) const { + return block_labels_ + block->GetBlockId(); + } + + virtual void Move(HInstruction* instruction, Location location, HInstruction* move_for) OVERRIDE; + + virtual size_t GetWordSize() const OVERRIDE { + return kArm64WordSize; + } + + virtual size_t FrameEntrySpillSize() const OVERRIDE; + + virtual HGraphVisitor* GetLocationBuilder() OVERRIDE { return &location_builder_; } + virtual HGraphVisitor* GetInstructionVisitor() OVERRIDE { return &instruction_visitor_; } + virtual Arm64Assembler* GetAssembler() OVERRIDE { return &assembler_; } + + // Emit a write barrier. + void MarkGCCard(vixl::Register object, vixl::Register value); + + // Register allocation. + + virtual void SetupBlockedRegisters() const OVERRIDE; + // AllocateFreeRegister() is only used when allocating registers locally + // during CompileBaseline(). + virtual Location AllocateFreeRegister(Primitive::Type type) const OVERRIDE; + + virtual Location GetStackLocation(HLoadLocal* load) const OVERRIDE; + + virtual size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE { + UNIMPLEMENTED(INFO) << "TODO: SaveCoreRegister"; + return 0; + } + + virtual size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE { + UNIMPLEMENTED(INFO) << "TODO: RestoreCoreRegister"; + return 0; + } + + // The number of registers that can be allocated. The register allocator may + // decide to reserve and not use a few of them. + // We do not consider registers sp, xzr, wzr. They are either not allocatable + // (xzr, wzr), or make for poor allocatable registers (sp alignment + // requirements, etc.). This also facilitates our task as all other registers + // can easily be mapped via to or from their type and index or code. + static const int kNumberOfAllocatableCoreRegisters = vixl::kNumberOfRegisters - 1; + static const int kNumberOfAllocatableFloatingPointRegisters = vixl::kNumberOfFPRegisters; + static const int kNumberOfAllocatableRegisters = + kNumberOfAllocatableCoreRegisters + kNumberOfAllocatableFloatingPointRegisters; + static constexpr int kNumberOfAllocatableRegisterPairs = 0; + + virtual void DumpCoreRegister(std::ostream& stream, int reg) const OVERRIDE; + virtual void DumpFloatingPointRegister(std::ostream& stream, int reg) const OVERRIDE; + + virtual InstructionSet GetInstructionSet() const OVERRIDE { + return InstructionSet::kArm64; + } + + void MoveHelper(Location destination, Location source, Primitive::Type type); + + virtual void Initialize() OVERRIDE { + HGraph* graph = GetGraph(); + int length = graph->GetBlocks().Size(); + block_labels_ = graph->GetArena()->AllocArray<vixl::Label>(length); + for (int i = 0; i < length; ++i) { + new(block_labels_ + i) vixl::Label(); + } + } + + private: + // Labels for each block that will be compiled. + vixl::Label* block_labels_; + + LocationsBuilderARM64 location_builder_; + InstructionCodeGeneratorARM64 instruction_visitor_; + Arm64Assembler assembler_; + + DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); +}; + +} // namespace arm64 +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 34fa46efd0..aa0f06b67f 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -56,7 +56,21 @@ class InvokeRuntimeCallingConvention : public CallingConvention<Register, XmmReg #define __ reinterpret_cast<X86Assembler*>(codegen->GetAssembler())-> -class NullCheckSlowPathX86 : public SlowPathCode { +class SlowPathCodeX86 : public SlowPathCode { + public: + SlowPathCodeX86() : entry_label_(), exit_label_() {} + + Label* GetEntryLabel() { return &entry_label_; } + Label* GetExitLabel() { return &exit_label_; } + + private: + Label entry_label_; + Label exit_label_; + + DISALLOW_COPY_AND_ASSIGN(SlowPathCodeX86); +}; + +class NullCheckSlowPathX86 : public SlowPathCodeX86 { public: explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {} @@ -71,7 +85,7 @@ class NullCheckSlowPathX86 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86); }; -class StackOverflowCheckSlowPathX86 : public SlowPathCode { +class StackOverflowCheckSlowPathX86 : public SlowPathCodeX86 { public: StackOverflowCheckSlowPathX86() {} @@ -86,7 +100,7 @@ class StackOverflowCheckSlowPathX86 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathX86); }; -class BoundsCheckSlowPathX86 : public SlowPathCode { +class BoundsCheckSlowPathX86 : public SlowPathCodeX86 { public: BoundsCheckSlowPathX86(HBoundsCheck* instruction, Location index_location, @@ -94,7 +108,7 @@ class BoundsCheckSlowPathX86 : public SlowPathCode { : instruction_(instruction), index_location_(index_location), length_location_(length_location) {} virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorX86* x86_codegen = reinterpret_cast<CodeGeneratorX86*>(codegen); + CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen); __ Bind(GetEntryLabel()); InvokeRuntimeCallingConvention calling_convention; x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), index_location_); @@ -111,12 +125,13 @@ class BoundsCheckSlowPathX86 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86); }; -class SuspendCheckSlowPathX86 : public SlowPathCode { +class SuspendCheckSlowPathX86 : public SlowPathCodeX86 { public: explicit SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor) : instruction_(instruction), successor_(successor) {} virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen); __ Bind(GetEntryLabel()); codegen->SaveLiveRegisters(instruction_->GetLocations()); __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend))); @@ -125,7 +140,7 @@ class SuspendCheckSlowPathX86 : public SlowPathCode { if (successor_ == nullptr) { __ jmp(GetReturnLabel()); } else { - __ jmp(codegen->GetLabelOf(successor_)); + __ jmp(x86_codegen->GetLabelOf(successor_)); } } @@ -167,16 +182,19 @@ void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) stream << X86ManagedRegister::FromXmmRegister(XmmRegister(reg)); } -void CodeGeneratorX86::SaveCoreRegister(Location stack_location, uint32_t reg_id) { - __ movl(Address(ESP, stack_location.GetStackIndex()), static_cast<Register>(reg_id)); +size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { + __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id)); + return kX86WordSize; } -void CodeGeneratorX86::RestoreCoreRegister(Location stack_location, uint32_t reg_id) { - __ movl(static_cast<Register>(reg_id), Address(ESP, stack_location.GetStackIndex())); +size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { + __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index)); + return kX86WordSize; } CodeGeneratorX86::CodeGeneratorX86(HGraph* graph) : CodeGenerator(graph, kNumberOfCpuRegisters, kNumberOfXmmRegisters, kNumberOfRegisterPairs), + block_labels_(graph->GetArena(), 0), location_builder_(graph, this), instruction_visitor_(graph, this), move_resolver_(graph->GetArena(), this) {} @@ -191,19 +209,11 @@ Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const { size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs); X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg)); + DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]); + DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]); blocked_core_registers_[pair.AsRegisterPairLow()] = true; blocked_core_registers_[pair.AsRegisterPairHigh()] = true; - // Block all other register pairs that share a register with `pair`. - for (int i = 0; i < kNumberOfRegisterPairs; i++) { - X86ManagedRegister current = - X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i)); - if (current.AsRegisterPairLow() == pair.AsRegisterPairLow() - || current.AsRegisterPairLow() == pair.AsRegisterPairHigh() - || current.AsRegisterPairHigh() == pair.AsRegisterPairLow() - || current.AsRegisterPairHigh() == pair.AsRegisterPairHigh()) { - blocked_register_pairs_[i] = true; - } - } + UpdateBlockedPairRegisters(); return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh()); } @@ -250,10 +260,19 @@ void CodeGeneratorX86::SetupBlockedRegisters() const { blocked_core_registers_[EBP] = true; blocked_core_registers_[ESI] = true; blocked_core_registers_[EDI] = true; - blocked_register_pairs_[EAX_EDI] = true; - blocked_register_pairs_[EDX_EDI] = true; - blocked_register_pairs_[ECX_EDI] = true; - blocked_register_pairs_[EBX_EDI] = true; + + UpdateBlockedPairRegisters(); +} + +void CodeGeneratorX86::UpdateBlockedPairRegisters() const { + for (int i = 0; i < kNumberOfRegisterPairs; i++) { + X86ManagedRegister current = + X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i)); + if (blocked_core_registers_[current.AsRegisterPairLow()] + || blocked_core_registers_[current.AsRegisterPairHigh()]) { + blocked_register_pairs_[i] = true; + } + } } InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen) @@ -276,7 +295,7 @@ void CodeGeneratorX86::GenerateFrameEntry() { __ subl(ESP, Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize)); if (!skip_overflow_check && kExplicitStackOverflowCheck) { - SlowPathCode* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86(); + SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86(); AddSlowPath(slow_path); __ fs()->cmpl(ESP, Address::Absolute(Thread::StackEndOffset<kX86WordSize>())); @@ -290,8 +309,8 @@ void CodeGeneratorX86::GenerateFrameExit() { __ addl(ESP, Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86WordSize)); } -void CodeGeneratorX86::Bind(Label* label) { - __ Bind(label); +void CodeGeneratorX86::Bind(HBasicBlock* block) { + __ Bind(GetLabelOf(block)); } void InstructionCodeGeneratorX86::LoadCurrentMethod(Register reg) { @@ -472,17 +491,23 @@ void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstr Immediate imm(instruction->AsIntConstant()->GetValue()); if (location.IsRegister()) { __ movl(location.As<Register>(), imm); - } else { + } else if (location.IsStackSlot()) { __ movl(Address(ESP, location.GetStackIndex()), imm); + } else { + DCHECK(location.IsConstant()); + DCHECK_EQ(location.GetConstant(), instruction); } } else if (instruction->IsLongConstant()) { int64_t value = instruction->AsLongConstant()->GetValue(); if (location.IsRegister()) { __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value))); __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value))); - } else { + } else if (location.IsDoubleStackSlot()) { __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value))); __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value))); + } else { + DCHECK(location.IsConstant()); + DCHECK_EQ(location.GetConstant(), instruction); } } else if (instruction->IsLoadLocal()) { int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal()); @@ -571,48 +596,66 @@ void LocationsBuilderX86::VisitIf(HIf* if_instr) { new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall); HInstruction* cond = if_instr->InputAt(0); if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) { - locations->SetInAt(0, Location::Any(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::Any()); } } void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) { HInstruction* cond = if_instr->InputAt(0); - bool materialized = !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization(); - // Moves do not affect the eflags register, so if the condition is evaluated - // just before the if, we don't need to evaluate it again. - bool eflags_set = cond->IsCondition() - && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr); - if (materialized) { - if (!eflags_set) { - // Materialized condition, compare against 0. - Location lhs = if_instr->GetLocations()->InAt(0); - if (lhs.IsRegister()) { - __ cmpl(lhs.As<Register>(), Immediate(0)); - } else { - __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0)); + if (cond->IsIntConstant()) { + // Constant condition, statically compared against 1. + int32_t cond_value = cond->AsIntConstant()->GetValue(); + if (cond_value == 1) { + if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), + if_instr->IfTrueSuccessor())) { + __ jmp(codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); } - __ j(kNotEqual, codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + return; } else { - __ j(X86Condition(cond->AsCondition()->GetCondition()), - codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + DCHECK_EQ(cond_value, 0); } } else { - Location lhs = cond->GetLocations()->InAt(0); - Location rhs = cond->GetLocations()->InAt(1); - // LHS is guaranteed to be in a register (see LocationsBuilderX86::VisitCondition). - if (rhs.IsRegister()) { - __ cmpl(lhs.As<Register>(), rhs.As<Register>()); - } else if (rhs.IsConstant()) { - HIntConstant* instruction = rhs.GetConstant()->AsIntConstant(); - Immediate imm(instruction->AsIntConstant()->GetValue()); - __ cmpl(lhs.As<Register>(), imm); + bool materialized = + !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization(); + // Moves do not affect the eflags register, so if the condition is + // evaluated just before the if, we don't need to evaluate it + // again. + bool eflags_set = cond->IsCondition() + && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr); + if (materialized) { + if (!eflags_set) { + // Materialized condition, compare against 0. + Location lhs = if_instr->GetLocations()->InAt(0); + if (lhs.IsRegister()) { + __ cmpl(lhs.As<Register>(), Immediate(0)); + } else { + __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0)); + } + __ j(kNotEqual, codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + } else { + __ j(X86Condition(cond->AsCondition()->GetCondition()), + codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + } } else { - __ cmpl(lhs.As<Register>(), Address(ESP, rhs.GetStackIndex())); + Location lhs = cond->GetLocations()->InAt(0); + Location rhs = cond->GetLocations()->InAt(1); + // LHS is guaranteed to be in a register (see + // LocationsBuilderX86::VisitCondition). + if (rhs.IsRegister()) { + __ cmpl(lhs.As<Register>(), rhs.As<Register>()); + } else if (rhs.IsConstant()) { + HIntConstant* instruction = rhs.GetConstant()->AsIntConstant(); + Immediate imm(instruction->AsIntConstant()->GetValue()); + __ cmpl(lhs.As<Register>(), imm); + } else { + __ cmpl(lhs.As<Register>(), Address(ESP, rhs.GetStackIndex())); + } + __ j(X86Condition(cond->AsCondition()->GetCondition()), + codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); } - __ j(X86Condition(cond->AsCondition()->GetCondition()), - codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); } - if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), if_instr->IfFalseSuccessor())) { + if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), + if_instr->IfFalseSuccessor())) { __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor())); } } @@ -664,8 +707,8 @@ void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store) { void LocationsBuilderX86::VisitCondition(HCondition* comp) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt(1, Location::Any(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::Any()); if (comp->NeedsMaterialization()) { locations->SetOut(Location::RequiresRegister()); } @@ -747,6 +790,7 @@ void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) { } void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant) { + // Will be generated at use site. } void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) { @@ -759,6 +803,26 @@ void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant) { // Will be generated at use site. } +void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant) { + // Will be generated at use site. +} + +void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant) { + // Will be generated at use site. +} + void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) { ret->SetLocations(nullptr); } @@ -921,6 +985,58 @@ void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } +void LocationsBuilderX86::VisitNeg(HNeg* neg) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); + switch (neg->GetResultType()) { + case Primitive::kPrimInt: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + break; + + default: + LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); + } +} + +void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) { + LocationSummary* locations = neg->GetLocations(); + Location out = locations->Out(); + Location in = locations->InAt(0); + switch (neg->GetResultType()) { + case Primitive::kPrimInt: + DCHECK(in.IsRegister()); + __ negl(out.As<Register>()); + break; + + case Primitive::kPrimLong: + DCHECK(in.IsRegisterPair()); + __ negl(out.AsRegisterPairLow<Register>()); + // Negation is similar to subtraction from zero. The least + // significant byte triggers a borrow when it is different from + // zero; to take it into account, add 1 to the most significant + // byte if the carry flag (CF) is set to 1 after the first NEGL + // operation. + __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0)); + __ negl(out.AsRegisterPairHigh<Register>()); + break; + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + break; + + default: + LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); + } +} + void LocationsBuilderX86::VisitAdd(HAdd* add) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); @@ -951,16 +1067,13 @@ void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) { LocationSummary* locations = add->GetLocations(); Location first = locations->InAt(0); Location second = locations->InAt(1); - + DCHECK(first.Equals(locations->Out())); switch (add->GetResultType()) { case Primitive::kPrimInt: { - DCHECK_EQ(first.As<Register>(), locations->Out().As<Register>()); if (second.IsRegister()) { __ addl(first.As<Register>(), second.As<Register>()); } else if (second.IsConstant()) { - HConstant* instruction = second.GetConstant(); - Immediate imm(instruction->AsIntConstant()->GetValue()); - __ addl(first.As<Register>(), imm); + __ addl(first.As<Register>(), Immediate(second.GetConstant()->AsIntConstant()->GetValue())); } else { __ addl(first.As<Register>(), Address(ESP, second.GetStackIndex())); } @@ -968,10 +1081,6 @@ void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) { } case Primitive::kPrimLong: { - DCHECK_EQ(first.AsRegisterPairLow<Register>(), - locations->Out().AsRegisterPairLow<Register>()); - DCHECK_EQ(first.AsRegisterPairHigh<Register>(), - locations->Out().AsRegisterPairHigh<Register>()); if (second.IsRegister()) { __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>()); __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>()); @@ -1017,16 +1126,16 @@ void LocationsBuilderX86::VisitSub(HSub* sub) { locations->SetOut(Location::SameAsFirstInput()); break; } - - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); break; + } default: - LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType(); + LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); } } @@ -1034,52 +1143,153 @@ void InstructionCodeGeneratorX86::VisitSub(HSub* sub) { LocationSummary* locations = sub->GetLocations(); Location first = locations->InAt(0); Location second = locations->InAt(1); + DCHECK(first.Equals(locations->Out())); switch (sub->GetResultType()) { case Primitive::kPrimInt: { - DCHECK_EQ(first.As<Register>(), - locations->Out().As<Register>()); if (second.IsRegister()) { - __ subl(first.As<Register>(), - second.As<Register>()); + __ subl(first.As<Register>(), second.As<Register>()); } else if (second.IsConstant()) { - HConstant* instruction = second.GetConstant(); - Immediate imm(instruction->AsIntConstant()->GetValue()); - __ subl(first.As<Register>(), imm); + __ subl(first.As<Register>(), Immediate(second.GetConstant()->AsIntConstant()->GetValue())); } else { - __ subl(first.As<Register>(), - Address(ESP, second.GetStackIndex())); + __ subl(first.As<Register>(), Address(ESP, second.GetStackIndex())); } break; } case Primitive::kPrimLong: { - DCHECK_EQ(first.AsRegisterPairLow<Register>(), - locations->Out().AsRegisterPairLow<Register>()); - DCHECK_EQ(first.AsRegisterPairHigh<Register>(), - locations->Out().AsRegisterPairHigh<Register>()); if (second.IsRegister()) { - __ subl(first.AsRegisterPairLow<Register>(), - second.AsRegisterPairLow<Register>()); - __ sbbl(first.AsRegisterPairHigh<Register>(), - second.AsRegisterPairHigh<Register>()); + __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>()); + __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>()); } else { - __ subl(first.AsRegisterPairLow<Register>(), - Address(ESP, second.GetStackIndex())); + __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex())); __ sbbl(first.AsRegisterPairHigh<Register>(), Address(ESP, second.GetHighStackIndex(kX86WordSize))); } break; } - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: + case Primitive::kPrimFloat: { + __ subss(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + case Primitive::kPrimDouble: { + __ subsd(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + default: LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); + } +} + +void LocationsBuilderX86::VisitMul(HMul* mul) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); + switch (mul->GetResultType()) { + case Primitive::kPrimInt: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::Any()); + locations->SetOut(Location::SameAsFirstInput()); break; + case Primitive::kPrimLong: { + locations->SetInAt(0, Location::RequiresRegister()); + // TODO: Currently this handles only stack operands: + // - we don't have enough registers because we currently use Quick ABI. + // - by the time we have a working register allocator we will probably change the ABI + // and fix the above. + // - we don't have a way yet to request operands on stack but the base line compiler + // will leave the operands on the stack with Any(). + locations->SetInAt(1, Location::Any()); + locations->SetOut(Location::SameAsFirstInput()); + // Needed for imul on 32bits with 64bits output. + locations->AddTemp(Location::RegisterLocation(EAX)); + locations->AddTemp(Location::RegisterLocation(EDX)); + break; + } + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + } default: - LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType(); + LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); + } +} + +void InstructionCodeGeneratorX86::VisitMul(HMul* mul) { + LocationSummary* locations = mul->GetLocations(); + Location first = locations->InAt(0); + Location second = locations->InAt(1); + DCHECK(first.Equals(locations->Out())); + + switch (mul->GetResultType()) { + case Primitive::kPrimInt: { + if (second.IsRegister()) { + __ imull(first.As<Register>(), second.As<Register>()); + } else if (second.IsConstant()) { + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue()); + __ imull(first.As<Register>(), imm); + } else { + DCHECK(second.IsStackSlot()); + __ imull(first.As<Register>(), Address(ESP, second.GetStackIndex())); + } + break; + } + + case Primitive::kPrimLong: { + DCHECK(second.IsDoubleStackSlot()); + + Register in1_hi = first.AsRegisterPairHigh<Register>(); + Register in1_lo = first.AsRegisterPairLow<Register>(); + Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize)); + Address in2_lo(ESP, second.GetStackIndex()); + Register eax = locations->GetTemp(0).As<Register>(); + Register edx = locations->GetTemp(1).As<Register>(); + + DCHECK_EQ(EAX, eax); + DCHECK_EQ(EDX, edx); + + // input: in1 - 64 bits, in2 - 64 bits + // output: in1 + // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo + // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] + // parts: in1.lo = (in1.lo * in2.lo)[31:0] + + __ movl(eax, in2_hi); + // eax <- in1.lo * in2.hi + __ imull(eax, in1_lo); + // in1.hi <- in1.hi * in2.lo + __ imull(in1_hi, in2_lo); + // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo + __ addl(in1_hi, eax); + // move in1_lo to eax to prepare for double precision + __ movl(eax, in1_lo); + // edx:eax <- in1.lo * in2.lo + __ mull(in2_lo); + // in1.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] + __ addl(in1_hi, edx); + // in1.lo <- (in1.lo * in2.lo)[31:0]; + __ movl(in1_lo, eax); + + break; + } + + case Primitive::kPrimFloat: { + __ mulss(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + case Primitive::kPrimDouble: { + __ mulsd(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + default: + LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); } } @@ -1104,6 +1314,28 @@ void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) { DCHECK(!codegen_->IsLeafMethod()); } +void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); + locations->SetOut(Location::RegisterLocation(EAX)); + InvokeRuntimeCallingConvention calling_convention; + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); +} + +void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) { + InvokeRuntimeCallingConvention calling_convention; + LoadCurrentMethod(calling_convention.GetRegisterAt(1)); + __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex())); + + __ fs()->call( + Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAllocArrayWithAccessCheck))); + + codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + DCHECK(!codegen_->IsLeafMethod()); +} + void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); @@ -1119,26 +1351,42 @@ void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) { void InstructionCodeGeneratorX86::VisitParameterValue(HParameterValue* instruction) { } -void LocationsBuilderX86::VisitNot(HNot* instruction) { +void LocationsBuilderX86::VisitNot(HNot* not_) { LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::SameAsFirstInput()); } -void InstructionCodeGeneratorX86::VisitNot(HNot* instruction) { - LocationSummary* locations = instruction->GetLocations(); +void InstructionCodeGeneratorX86::VisitNot(HNot* not_) { + LocationSummary* locations = not_->GetLocations(); + DCHECK_EQ(locations->InAt(0).As<Register>(), locations->Out().As<Register>()); Location out = locations->Out(); DCHECK_EQ(locations->InAt(0).As<Register>(), out.As<Register>()); - __ xorl(out.As<Register>(), Immediate(1)); + switch (not_->InputAt(0)->GetType()) { + case Primitive::kPrimBoolean: + __ xorl(out.As<Register>(), Immediate(1)); + break; + + case Primitive::kPrimInt: + __ notl(out.As<Register>()); + break; + + case Primitive::kPrimLong: + LOG(FATAL) << "Not yet implemented type for not operation " << not_->GetResultType(); + break; + + default: + LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); + } } void LocationsBuilderX86::VisitCompare(HCompare* compare) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt(1, Location::Any(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::Any()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) { @@ -1207,12 +1455,11 @@ void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) || (field_type == Primitive::kPrimByte); // The register allocator does not support multiple // inputs that die at entry with one in a specific register. - bool dies_at_entry = !is_object_type && !is_byte_type; if (is_byte_type) { // Ensure the value is in a byte register. - locations->SetInAt(1, Location::RegisterLocation(EAX), dies_at_entry); + locations->SetInAt(1, Location::RegisterLocation(EAX)); } else { - locations->SetInAt(1, Location::RequiresRegister(), dies_at_entry); + locations->SetInAt(1, Location::RequiresRegister()); } // Temporary registers for the write barrier. if (is_object_type) { @@ -1288,8 +1535,8 @@ void CodeGeneratorX86::MarkGCCard(Register temp, Register card, Register object, void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { @@ -1356,7 +1603,7 @@ void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) { } void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) { - SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction); + SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction); codegen_->AddSlowPath(slow_path); LocationSummary* locations = instruction->GetLocations(); @@ -1378,10 +1625,9 @@ void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) { void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt( - 1, Location::RegisterOrConstant(instruction->InputAt(1)), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) { @@ -1494,16 +1740,13 @@ void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) { // We need the inputs to be different than the output in case of long operation. // In case of a byte operation, the register allocator does not support multiple // inputs that die at entry with one in a specific register. - bool dies_at_entry = value_type != Primitive::kPrimLong && !is_byte_type; - locations->SetInAt(0, Location::RequiresRegister(), dies_at_entry); - locations->SetInAt( - 1, Location::RegisterOrConstant(instruction->InputAt(1)), dies_at_entry); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); if (is_byte_type) { // Ensure the value is in a byte register. - locations->SetInAt(2, Location::ByteRegisterOrConstant( - EAX, instruction->InputAt(2)), dies_at_entry); + locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2))); } else { - locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)), dies_at_entry); + locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2))); } } } @@ -1633,8 +1876,8 @@ void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) { void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); instruction->SetLocations(locations); } @@ -1658,7 +1901,7 @@ void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) { void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) { LocationSummary* locations = instruction->GetLocations(); - SlowPathCode* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86( + SlowPathCodeX86* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86( instruction, locations->InAt(0), locations->InAt(1)); codegen_->AddSlowPath(slow_path); diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index a1a72a2bd7..fff91d179a 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -142,10 +142,10 @@ class CodeGeneratorX86 : public CodeGenerator { virtual void GenerateFrameEntry() OVERRIDE; virtual void GenerateFrameExit() OVERRIDE; - virtual void Bind(Label* label) OVERRIDE; + virtual void Bind(HBasicBlock* block) OVERRIDE; virtual void Move(HInstruction* instruction, Location location, HInstruction* move_for) OVERRIDE; - virtual void SaveCoreRegister(Location stack_location, uint32_t reg_id) OVERRIDE; - virtual void RestoreCoreRegister(Location stack_location, uint32_t reg_id) OVERRIDE; + virtual size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + virtual size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; virtual size_t GetWordSize() const OVERRIDE { return kX86WordSize; @@ -166,6 +166,7 @@ class CodeGeneratorX86 : public CodeGenerator { } virtual void SetupBlockedRegisters() const OVERRIDE; + virtual Location AllocateFreeRegister(Primitive::Type type) const OVERRIDE; virtual Location GetStackLocation(HLoadLocal* load) const OVERRIDE; @@ -173,6 +174,9 @@ class CodeGeneratorX86 : public CodeGenerator { virtual void DumpCoreRegister(std::ostream& stream, int reg) const OVERRIDE; virtual void DumpFloatingPointRegister(std::ostream& stream, int reg) const OVERRIDE; + // Blocks all register pairs made out of blocked core registers. + void UpdateBlockedPairRegisters() const; + ParallelMoveResolverX86* GetMoveResolver() { return &move_resolver_; } @@ -189,7 +193,17 @@ class CodeGeneratorX86 : public CodeGenerator { // Emit a write barrier. void MarkGCCard(Register temp, Register card, Register object, Register value); + Label* GetLabelOf(HBasicBlock* block) const { + return block_labels_.GetRawStorage() + block->GetBlockId(); + } + + virtual void Initialize() OVERRIDE { + block_labels_.SetSize(GetGraph()->GetBlocks().Size()); + } + private: + // Labels for each block that will be compiled. + GrowableArray<Label> block_labels_; LocationsBuilderX86 location_builder_; InstructionCodeGeneratorX86 instruction_visitor_; ParallelMoveResolverX86 move_resolver_; diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 059140d9bf..892ca9d9c7 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -60,7 +60,21 @@ class InvokeRuntimeCallingConvention : public CallingConvention<Register, FloatR #define __ reinterpret_cast<X86_64Assembler*>(codegen->GetAssembler())-> -class NullCheckSlowPathX86_64 : public SlowPathCode { +class SlowPathCodeX86_64 : public SlowPathCode { + public: + SlowPathCodeX86_64() : entry_label_(), exit_label_() {} + + Label* GetEntryLabel() { return &entry_label_; } + Label* GetExitLabel() { return &exit_label_; } + + private: + Label entry_label_; + Label exit_label_; + + DISALLOW_COPY_AND_ASSIGN(SlowPathCodeX86_64); +}; + +class NullCheckSlowPathX86_64 : public SlowPathCodeX86_64 { public: explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {} @@ -76,7 +90,7 @@ class NullCheckSlowPathX86_64 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64); }; -class StackOverflowCheckSlowPathX86_64 : public SlowPathCode { +class StackOverflowCheckSlowPathX86_64 : public SlowPathCodeX86_64 { public: StackOverflowCheckSlowPathX86_64() {} @@ -92,12 +106,13 @@ class StackOverflowCheckSlowPathX86_64 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(StackOverflowCheckSlowPathX86_64); }; -class SuspendCheckSlowPathX86_64 : public SlowPathCode { +class SuspendCheckSlowPathX86_64 : public SlowPathCodeX86_64 { public: explicit SuspendCheckSlowPathX86_64(HSuspendCheck* instruction, HBasicBlock* successor) : instruction_(instruction), successor_(successor) {} virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen); __ Bind(GetEntryLabel()); codegen->SaveLiveRegisters(instruction_->GetLocations()); __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pTestSuspend), true)); @@ -106,7 +121,7 @@ class SuspendCheckSlowPathX86_64 : public SlowPathCode { if (successor_ == nullptr) { __ jmp(GetReturnLabel()); } else { - __ jmp(codegen->GetLabelOf(successor_)); + __ jmp(x64_codegen->GetLabelOf(successor_)); } } @@ -123,7 +138,7 @@ class SuspendCheckSlowPathX86_64 : public SlowPathCode { DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86_64); }; -class BoundsCheckSlowPathX86_64 : public SlowPathCode { +class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 { public: BoundsCheckSlowPathX86_64(HBoundsCheck* instruction, Location index_location, @@ -133,7 +148,7 @@ class BoundsCheckSlowPathX86_64 : public SlowPathCode { length_location_(length_location) {} virtual void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorX86_64* x64_codegen = reinterpret_cast<CodeGeneratorX86_64*>(codegen); + CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen); __ Bind(GetEntryLabel()); InvokeRuntimeCallingConvention calling_convention; x64_codegen->Move(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), index_location_); @@ -176,16 +191,29 @@ void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int re stream << X86_64ManagedRegister::FromXmmRegister(FloatRegister(reg)); } -void CodeGeneratorX86_64::SaveCoreRegister(Location stack_location, uint32_t reg_id) { - __ movq(Address(CpuRegister(RSP), stack_location.GetStackIndex()), CpuRegister(reg_id)); +size_t CodeGeneratorX86_64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { + __ movq(Address(CpuRegister(RSP), stack_index), CpuRegister(reg_id)); + return kX86_64WordSize; +} + +size_t CodeGeneratorX86_64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { + __ movq(CpuRegister(reg_id), Address(CpuRegister(RSP), stack_index)); + return kX86_64WordSize; } -void CodeGeneratorX86_64::RestoreCoreRegister(Location stack_location, uint32_t reg_id) { - __ movq(CpuRegister(reg_id), Address(CpuRegister(RSP), stack_location.GetStackIndex())); +size_t CodeGeneratorX86_64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { + __ movsd(Address(CpuRegister(RSP), stack_index), XmmRegister(reg_id)); + return kX86_64WordSize; +} + +size_t CodeGeneratorX86_64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { + __ movsd(XmmRegister(reg_id), Address(CpuRegister(RSP), stack_index)); + return kX86_64WordSize; } CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph) : CodeGenerator(graph, kNumberOfCpuRegisters, kNumberOfFloatRegisters, 0), + block_labels_(graph->GetArena(), 0), location_builder_(graph, this), instruction_visitor_(graph, this), move_resolver_(graph->GetArena(), this) {} @@ -266,7 +294,7 @@ void CodeGeneratorX86_64::GenerateFrameEntry() { Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize)); if (!skip_overflow_check && kExplicitStackOverflowCheck) { - SlowPathCode* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86_64(); + SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) StackOverflowCheckSlowPathX86_64(); AddSlowPath(slow_path); __ gs()->cmpq(CpuRegister(RSP), @@ -282,8 +310,8 @@ void CodeGeneratorX86_64::GenerateFrameExit() { Immediate(GetFrameSize() - kNumberOfPushedRegistersAtEntry * kX86_64WordSize)); } -void CodeGeneratorX86_64::Bind(Label* label) { - __ Bind(label); +void CodeGeneratorX86_64::Bind(HBasicBlock* block) { + __ Bind(GetLabelOf(block)); } void InstructionCodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) { @@ -379,16 +407,22 @@ void CodeGeneratorX86_64::Move(HInstruction* instruction, Immediate imm(instruction->AsIntConstant()->GetValue()); if (location.IsRegister()) { __ movl(location.As<CpuRegister>(), imm); - } else { + } else if (location.IsStackSlot()) { __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm); + } else { + DCHECK(location.IsConstant()); + DCHECK_EQ(location.GetConstant(), instruction); } } else if (instruction->IsLongConstant()) { int64_t value = instruction->AsLongConstant()->GetValue(); if (location.IsRegister()) { __ movq(location.As<CpuRegister>(), Immediate(value)); - } else { + } else if (location.IsDoubleStackSlot()) { __ movq(CpuRegister(TMP), Immediate(value)); __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP)); + } else { + DCHECK(location.IsConstant()); + DCHECK_EQ(location.GetConstant(), instruction); } } else if (instruction->IsLoadLocal()) { switch (instruction->GetType()) { @@ -473,46 +507,65 @@ void LocationsBuilderX86_64::VisitIf(HIf* if_instr) { new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall); HInstruction* cond = if_instr->InputAt(0); if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) { - locations->SetInAt(0, Location::Any(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::Any()); } } void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) { HInstruction* cond = if_instr->InputAt(0); - bool materialized = !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization(); - // Moves do not affect the eflags register, so if the condition is evaluated - // just before the if, we don't need to evaluate it again. - bool eflags_set = cond->IsCondition() - && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr); - if (materialized) { - if (!eflags_set) { - // Materialized condition, compare against 0. - Location lhs = if_instr->GetLocations()->InAt(0); - if (lhs.IsRegister()) { - __ cmpl(lhs.As<CpuRegister>(), Immediate(0)); - } else { - __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()), Immediate(0)); + if (cond->IsIntConstant()) { + // Constant condition, statically compared against 1. + int32_t cond_value = cond->AsIntConstant()->GetValue(); + if (cond_value == 1) { + if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), + if_instr->IfTrueSuccessor())) { + __ jmp(codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); } - __ j(kNotEqual, codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + return; } else { - __ j(X86_64Condition(cond->AsCondition()->GetCondition()), - codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + DCHECK_EQ(cond_value, 0); } } else { - Location lhs = cond->GetLocations()->InAt(0); - Location rhs = cond->GetLocations()->InAt(1); - if (rhs.IsRegister()) { - __ cmpl(lhs.As<CpuRegister>(), rhs.As<CpuRegister>()); - } else if (rhs.IsConstant()) { - __ cmpl(lhs.As<CpuRegister>(), - Immediate(rhs.GetConstant()->AsIntConstant()->GetValue())); + bool materialized = + !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization(); + // Moves do not affect the eflags register, so if the condition is + // evaluated just before the if, we don't need to evaluate it + // again. + bool eflags_set = cond->IsCondition() + && cond->AsCondition()->IsBeforeWhenDisregardMoves(if_instr); + if (materialized) { + if (!eflags_set) { + // Materialized condition, compare against 0. + Location lhs = if_instr->GetLocations()->InAt(0); + if (lhs.IsRegister()) { + __ cmpl(lhs.As<CpuRegister>(), Immediate(0)); + } else { + __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()), + Immediate(0)); + } + __ j(kNotEqual, codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + } else { + __ j(X86_64Condition(cond->AsCondition()->GetCondition()), + codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); + } } else { - __ cmpl(lhs.As<CpuRegister>(), Address(CpuRegister(RSP), rhs.GetStackIndex())); + Location lhs = cond->GetLocations()->InAt(0); + Location rhs = cond->GetLocations()->InAt(1); + if (rhs.IsRegister()) { + __ cmpl(lhs.As<CpuRegister>(), rhs.As<CpuRegister>()); + } else if (rhs.IsConstant()) { + __ cmpl(lhs.As<CpuRegister>(), + Immediate(rhs.GetConstant()->AsIntConstant()->GetValue())); + } else { + __ cmpl(lhs.As<CpuRegister>(), + Address(CpuRegister(RSP), rhs.GetStackIndex())); + } + __ j(X86_64Condition(cond->AsCondition()->GetCondition()), + codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); } - __ j(X86_64Condition(cond->AsCondition()->GetCondition()), - codegen_->GetLabelOf(if_instr->IfTrueSuccessor())); } - if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), if_instr->IfFalseSuccessor())) { + if (!codegen_->GoesToNextBlock(if_instr->GetBlock(), + if_instr->IfFalseSuccessor())) { __ jmp(codegen_->GetLabelOf(if_instr->IfFalseSuccessor())); } } @@ -563,8 +616,8 @@ void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) { void LocationsBuilderX86_64::VisitCondition(HCondition* comp) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt(1, Location::Any(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::Any()); if (comp->NeedsMaterialization()) { locations->SetOut(Location::RequiresRegister()); } @@ -641,9 +694,9 @@ void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual void LocationsBuilderX86_64::VisitCompare(HCompare* compare) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetInAt(1, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) { @@ -679,6 +732,7 @@ void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) { } void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) { + // Will be generated at use site. } void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) { @@ -688,6 +742,27 @@ void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) { } void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) { + // Will be generated at use site. +} + +void LocationsBuilderX86_64::VisitFloatConstant(HFloatConstant* constant) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorX86_64::VisitFloatConstant(HFloatConstant* constant) { + // Will be generated at use site. +} + +void LocationsBuilderX86_64::VisitDoubleConstant(HDoubleConstant* constant) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); + locations->SetOut(Location::ConstantLocation(constant)); +} + +void InstructionCodeGeneratorX86_64::VisitDoubleConstant(HDoubleConstant* constant) { + // Will be generated at use site. } void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) { @@ -896,6 +971,51 @@ void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } +void LocationsBuilderX86_64::VisitNeg(HNeg* neg) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); + switch (neg->GetResultType()) { + case Primitive::kPrimInt: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + break; + + default: + LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); + } +} + +void InstructionCodeGeneratorX86_64::VisitNeg(HNeg* neg) { + LocationSummary* locations = neg->GetLocations(); + Location out = locations->Out(); + Location in = locations->InAt(0); + switch (neg->GetResultType()) { + case Primitive::kPrimInt: + DCHECK(in.IsRegister()); + __ negl(out.As<CpuRegister>()); + break; + + case Primitive::kPrimLong: + DCHECK(in.IsRegister()); + __ negq(out.As<CpuRegister>()); + break; + + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + LOG(FATAL) << "Not yet implemented neg type " << neg->GetResultType(); + break; + + default: + LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); + } +} + void LocationsBuilderX86_64::VisitAdd(HAdd* add) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); @@ -917,7 +1037,7 @@ void LocationsBuilderX86_64::VisitAdd(HAdd* add) { case Primitive::kPrimDouble: case Primitive::kPrimFloat: { locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::Any()); + locations->SetInAt(1, Location::RequiresFpuRegister()); locations->SetOut(Location::SameAsFirstInput()); break; } @@ -931,19 +1051,17 @@ void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) { LocationSummary* locations = add->GetLocations(); Location first = locations->InAt(0); Location second = locations->InAt(1); - DCHECK(first.Equals(locations->Out())); + switch (add->GetResultType()) { case Primitive::kPrimInt: { if (second.IsRegister()) { __ addl(first.As<CpuRegister>(), second.As<CpuRegister>()); } else if (second.IsConstant()) { - HConstant* instruction = second.GetConstant(); - Immediate imm(instruction->AsIntConstant()->GetValue()); + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue()); __ addl(first.As<CpuRegister>(), imm); } else { - __ addl(first.As<CpuRegister>(), - Address(CpuRegister(RSP), second.GetStackIndex())); + __ addl(first.As<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex())); } break; } @@ -954,21 +1072,12 @@ void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) { } case Primitive::kPrimFloat: { - if (second.IsFpuRegister()) { - __ addss(first.As<XmmRegister>(), second.As<XmmRegister>()); - } else { - __ addss(first.As<XmmRegister>(), - Address(CpuRegister(RSP), second.GetStackIndex())); - } + __ addss(first.As<XmmRegister>(), second.As<XmmRegister>()); break; } case Primitive::kPrimDouble: { - if (second.IsFpuRegister()) { - __ addsd(first.As<XmmRegister>(), second.As<XmmRegister>()); - } else { - __ addsd(first.As<XmmRegister>(), Address(CpuRegister(RSP), second.GetStackIndex())); - } + __ addsd(first.As<XmmRegister>(), second.As<XmmRegister>()); break; } @@ -993,53 +1102,119 @@ void LocationsBuilderX86_64::VisitSub(HSub* sub) { locations->SetOut(Location::SameAsFirstInput()); break; } - - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); break; - + } default: - LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType(); + LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); } } void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) { LocationSummary* locations = sub->GetLocations(); - DCHECK_EQ(locations->InAt(0).As<CpuRegister>().AsRegister(), - locations->Out().As<CpuRegister>().AsRegister()); + Location first = locations->InAt(0); + Location second = locations->InAt(1); + DCHECK(first.Equals(locations->Out())); switch (sub->GetResultType()) { case Primitive::kPrimInt: { - if (locations->InAt(1).IsRegister()) { - __ subl(locations->InAt(0).As<CpuRegister>(), - locations->InAt(1).As<CpuRegister>()); - } else if (locations->InAt(1).IsConstant()) { - HConstant* instruction = locations->InAt(1).GetConstant(); - Immediate imm(instruction->AsIntConstant()->GetValue()); - __ subl(locations->InAt(0).As<CpuRegister>(), imm); + if (second.IsRegister()) { + __ subl(first.As<CpuRegister>(), second.As<CpuRegister>()); + } else if (second.IsConstant()) { + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue()); + __ subl(first.As<CpuRegister>(), imm); } else { - __ subl(locations->InAt(0).As<CpuRegister>(), - Address(CpuRegister(RSP), locations->InAt(1).GetStackIndex())); + __ subl(first.As<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex())); } break; } case Primitive::kPrimLong: { - __ subq(locations->InAt(0).As<CpuRegister>(), - locations->InAt(1).As<CpuRegister>()); + __ subq(first.As<CpuRegister>(), second.As<CpuRegister>()); break; } - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: + case Primitive::kPrimFloat: { + __ subss(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + case Primitive::kPrimDouble: { + __ subsd(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + default: LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); + } +} + +void LocationsBuilderX86_64::VisitMul(HMul* mul) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); + switch (mul->GetResultType()) { + case Primitive::kPrimInt: { + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::Any()); + locations->SetOut(Location::SameAsFirstInput()); + break; + } + case Primitive::kPrimLong: { + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); break; + } + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + } default: - LOG(FATAL) << "Unimplemented sub type " << sub->GetResultType(); + LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); + } +} + +void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) { + LocationSummary* locations = mul->GetLocations(); + Location first = locations->InAt(0); + Location second = locations->InAt(1); + DCHECK(first.Equals(locations->Out())); + switch (mul->GetResultType()) { + case Primitive::kPrimInt: { + if (second.IsRegister()) { + __ imull(first.As<CpuRegister>(), second.As<CpuRegister>()); + } else if (second.IsConstant()) { + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue()); + __ imull(first.As<CpuRegister>(), imm); + } else { + DCHECK(second.IsStackSlot()); + __ imull(first.As<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex())); + } + break; + } + case Primitive::kPrimLong: { + __ imulq(first.As<CpuRegister>(), second.As<CpuRegister>()); + break; + } + + case Primitive::kPrimFloat: { + __ mulss(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + case Primitive::kPrimDouble: { + __ mulsd(first.As<XmmRegister>(), second.As<XmmRegister>()); + break; + } + + default: + LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); } } @@ -1064,6 +1239,28 @@ void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); } +void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) { + LocationSummary* locations = + new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall); + InvokeRuntimeCallingConvention calling_convention; + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->SetOut(Location::RegisterLocation(RAX)); + locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); +} + +void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) { + InvokeRuntimeCallingConvention calling_convention; + LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1))); + __ movq(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction->GetTypeIndex())); + + __ gs()->call(Address::Absolute( + QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocArrayWithAccessCheck), true)); + + DCHECK(!codegen_->IsLeafMethod()); + codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); +} + void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); @@ -1080,18 +1277,34 @@ void InstructionCodeGeneratorX86_64::VisitParameterValue(HParameterValue* instru // Nothing to do, the parameter is already at its location. } -void LocationsBuilderX86_64::VisitNot(HNot* instruction) { +void LocationsBuilderX86_64::VisitNot(HNot* not_) { LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::SameAsFirstInput()); } -void InstructionCodeGeneratorX86_64::VisitNot(HNot* instruction) { - LocationSummary* locations = instruction->GetLocations(); +void InstructionCodeGeneratorX86_64::VisitNot(HNot* not_) { + LocationSummary* locations = not_->GetLocations(); DCHECK_EQ(locations->InAt(0).As<CpuRegister>().AsRegister(), locations->Out().As<CpuRegister>().AsRegister()); - __ xorq(locations->Out().As<CpuRegister>(), Immediate(1)); + Location out = locations->Out(); + switch (not_->InputAt(0)->GetType()) { + case Primitive::kPrimBoolean: + __ xorq(out.As<CpuRegister>(), Immediate(1)); + break; + + case Primitive::kPrimInt: + __ notl(out.As<CpuRegister>()); + break; + + case Primitive::kPrimLong: + LOG(FATAL) << "Not yet implemented type for not operation " << not_->GetResultType(); + break; + + default: + LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); + } } void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) { @@ -1112,9 +1325,8 @@ void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instructio new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); Primitive::Type field_type = instruction->GetFieldType(); bool is_object_type = field_type == Primitive::kPrimNot; - bool dies_at_entry = !is_object_type; - locations->SetInAt(0, Location::RequiresRegister(), dies_at_entry); - locations->SetInAt(1, Location::RequiresRegister(), dies_at_entry); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); if (is_object_type) { // Temporary registers for the write barrier. locations->AddTemp(Location::RequiresRegister()); @@ -1171,8 +1383,8 @@ void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* in void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { @@ -1233,7 +1445,7 @@ void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) { } void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) { - SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction); + SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction); codegen_->AddSlowPath(slow_path); LocationSummary* locations = instruction->GetLocations(); @@ -1255,10 +1467,10 @@ void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) { void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt( - 1, Location::RegisterOrConstant(instruction->InputAt(1)), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + 1, Location::RegisterOrConstant(instruction->InputAt(1))); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { @@ -1341,10 +1553,30 @@ void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) { break; } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - LOG(FATAL) << "Unimplemented register type " << instruction->GetType(); - UNREACHABLE(); + case Primitive::kPrimFloat: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value(); + XmmRegister out = locations->Out().As<XmmRegister>(); + if (index.IsConstant()) { + __ movss(out, Address(obj, + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset)); + } else { + __ movss(out, Address(obj, index.As<CpuRegister>(), TIMES_4, data_offset)); + } + break; + } + + case Primitive::kPrimDouble: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value(); + XmmRegister out = locations->Out().As<XmmRegister>(); + if (index.IsConstant()) { + __ movsd(out, Address(obj, + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset)); + } else { + __ movsd(out, Address(obj, index.As<CpuRegister>(), TIMES_8, data_offset)); + } + break; + } + case Primitive::kPrimVoid: LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); @@ -1362,14 +1594,16 @@ void LocationsBuilderX86_64::VisitArraySet(HArraySet* instruction) { locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); } else { - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); + locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt( - 1, Location::RegisterOrConstant(instruction->InputAt(1)), Location::kDiesAtEntry); - locations->SetInAt(2, Location::RequiresRegister(), Location::kDiesAtEntry); + 1, Location::RegisterOrConstant(instruction->InputAt(1))); + locations->SetInAt(2, Location::RequiresRegister()); if (value_type == Primitive::kPrimLong) { - locations->SetInAt(2, Location::RequiresRegister(), Location::kDiesAtEntry); + locations->SetInAt(2, Location::RequiresRegister()); + } else if (value_type == Primitive::kPrimFloat || value_type == Primitive::kPrimDouble) { + locations->SetInAt(2, Location::RequiresFpuRegister()); } else { - locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)), Location::kDiesAtEntry); + locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2))); } } } @@ -1440,6 +1674,7 @@ void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) { __ movl(Address(obj, index.As<CpuRegister>(), TIMES_4, data_offset), value.As<CpuRegister>()); } else { + DCHECK(value.IsConstant()) << value; __ movl(Address(obj, index.As<CpuRegister>(), TIMES_4, data_offset), Immediate(value.GetConstant()->AsIntConstant()->GetValue())); } @@ -1468,10 +1703,34 @@ void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) { break; } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - LOG(FATAL) << "Unimplemented register type " << instruction->GetType(); - UNREACHABLE(); + case Primitive::kPrimFloat: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value(); + if (index.IsConstant()) { + size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + DCHECK(value.IsFpuRegister()); + __ movss(Address(obj, offset), value.As<XmmRegister>()); + } else { + DCHECK(value.IsFpuRegister()); + __ movss(Address(obj, index.As<CpuRegister>(), TIMES_4, data_offset), + value.As<XmmRegister>()); + } + break; + } + + case Primitive::kPrimDouble: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value(); + if (index.IsConstant()) { + size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; + DCHECK(value.IsFpuRegister()); + __ movsd(Address(obj, offset), value.As<XmmRegister>()); + } else { + DCHECK(value.IsFpuRegister()); + __ movsd(Address(obj, index.As<CpuRegister>(), TIMES_8, data_offset), + value.As<XmmRegister>()); + } + break; + } + case Primitive::kPrimVoid: LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); @@ -1481,8 +1740,8 @@ void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) { void LocationsBuilderX86_64::VisitArrayLength(HArrayLength* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister(), Location::kDiesAtEntry); - locations->SetOut(Location::RequiresRegister()); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); } void InstructionCodeGeneratorX86_64::VisitArrayLength(HArrayLength* instruction) { @@ -1505,7 +1764,7 @@ void LocationsBuilderX86_64::VisitBoundsCheck(HBoundsCheck* instruction) { void InstructionCodeGeneratorX86_64::VisitBoundsCheck(HBoundsCheck* instruction) { LocationSummary* locations = instruction->GetLocations(); - SlowPathCode* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64( + SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64( instruction, locations->InAt(0), locations->InAt(1)); codegen_->AddSlowPath(slow_path); @@ -1605,6 +1864,9 @@ void ParallelMoveResolverX86_64::EmitMove(size_t index) { if (destination.IsRegister()) { __ movl(destination.As<CpuRegister>(), Address(CpuRegister(RSP), source.GetStackIndex())); + } else if (destination.IsFpuRegister()) { + __ movss(destination.As<XmmRegister>(), + Address(CpuRegister(RSP), source.GetStackIndex())); } else { DCHECK(destination.IsStackSlot()); __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex())); @@ -1614,8 +1876,10 @@ void ParallelMoveResolverX86_64::EmitMove(size_t index) { if (destination.IsRegister()) { __ movq(destination.As<CpuRegister>(), Address(CpuRegister(RSP), source.GetStackIndex())); + } else if (destination.IsFpuRegister()) { + __ movsd(destination.As<XmmRegister>(), Address(CpuRegister(RSP), source.GetStackIndex())); } else { - DCHECK(destination.IsDoubleStackSlot()); + DCHECK(destination.IsDoubleStackSlot()) << destination; __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex())); __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP)); } @@ -1626,6 +1890,7 @@ void ParallelMoveResolverX86_64::EmitMove(size_t index) { if (destination.IsRegister()) { __ movl(destination.As<CpuRegister>(), imm); } else { + DCHECK(destination.IsStackSlot()) << destination; __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm); } } else if (constant->IsLongConstant()) { @@ -1633,14 +1898,42 @@ void ParallelMoveResolverX86_64::EmitMove(size_t index) { if (destination.IsRegister()) { __ movq(destination.As<CpuRegister>(), Immediate(value)); } else { + DCHECK(destination.IsDoubleStackSlot()) << destination; __ movq(CpuRegister(TMP), Immediate(value)); __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP)); } + } else if (constant->IsFloatConstant()) { + Immediate imm(bit_cast<float, int32_t>(constant->AsFloatConstant()->GetValue())); + if (destination.IsFpuRegister()) { + __ movl(CpuRegister(TMP), imm); + __ movd(destination.As<XmmRegister>(), CpuRegister(TMP)); + } else { + DCHECK(destination.IsStackSlot()) << destination; + __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm); + } + } else { + DCHECK(constant->IsDoubleConstant()) << constant->DebugName(); + Immediate imm(bit_cast<double, int64_t>(constant->AsDoubleConstant()->GetValue())); + if (destination.IsFpuRegister()) { + __ movq(CpuRegister(TMP), imm); + __ movd(destination.As<XmmRegister>(), CpuRegister(TMP)); + } else { + DCHECK(destination.IsDoubleStackSlot()) << destination; + __ movq(CpuRegister(TMP), imm); + __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP)); + } + } + } else if (source.IsFpuRegister()) { + if (destination.IsFpuRegister()) { + __ movaps(destination.As<XmmRegister>(), source.As<XmmRegister>()); + } else if (destination.IsStackSlot()) { + __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()), + source.As<XmmRegister>()); } else { - LOG(FATAL) << "Unimplemented constant type"; + DCHECK(destination.IsDoubleStackSlot()); + __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()), + source.As<XmmRegister>()); } - } else { - LOG(FATAL) << "Unimplemented"; } } @@ -1682,6 +1975,18 @@ void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) { CpuRegister(ensure_scratch.GetRegister())); } +void ParallelMoveResolverX86_64::Exchange32(XmmRegister reg, int mem) { + __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem)); + __ movss(Address(CpuRegister(RSP), mem), reg); + __ movd(reg, CpuRegister(TMP)); +} + +void ParallelMoveResolverX86_64::Exchange64(XmmRegister reg, int mem) { + __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem)); + __ movsd(Address(CpuRegister(RSP), mem), reg); + __ movd(reg, CpuRegister(TMP)); +} + void ParallelMoveResolverX86_64::EmitSwap(size_t index) { MoveOperands* move = moves_.Get(index); Location source = move->GetSource(); @@ -1701,8 +2006,20 @@ void ParallelMoveResolverX86_64::EmitSwap(size_t index) { Exchange64(destination.As<CpuRegister>(), source.GetStackIndex()); } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { Exchange64(destination.GetStackIndex(), source.GetStackIndex()); + } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { + __ movd(CpuRegister(TMP), source.As<XmmRegister>()); + __ movaps(source.As<XmmRegister>(), destination.As<XmmRegister>()); + __ movd(destination.As<XmmRegister>(), CpuRegister(TMP)); + } else if (source.IsFpuRegister() && destination.IsStackSlot()) { + Exchange32(source.As<XmmRegister>(), destination.GetStackIndex()); + } else if (source.IsStackSlot() && destination.IsFpuRegister()) { + Exchange32(destination.As<XmmRegister>(), source.GetStackIndex()); + } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) { + Exchange64(source.As<XmmRegister>(), destination.GetStackIndex()); + } else if (source.IsDoubleStackSlot() && destination.IsFpuRegister()) { + Exchange64(destination.As<XmmRegister>(), source.GetStackIndex()); } else { - LOG(FATAL) << "Unimplemented"; + LOG(FATAL) << "Unimplemented swap between " << source << " and " << destination; } } diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 288f3f61f9..e04a8d8ab9 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -80,8 +80,10 @@ class ParallelMoveResolverX86_64 : public ParallelMoveResolver { private: void Exchange32(CpuRegister reg, int mem); + void Exchange32(XmmRegister reg, int mem); void Exchange32(int mem1, int mem2); void Exchange64(CpuRegister reg, int mem); + void Exchange64(XmmRegister reg, int mem); void Exchange64(int mem1, int mem2); CodeGeneratorX86_64* const codegen_; @@ -144,10 +146,12 @@ class CodeGeneratorX86_64 : public CodeGenerator { virtual void GenerateFrameEntry() OVERRIDE; virtual void GenerateFrameExit() OVERRIDE; - virtual void Bind(Label* label) OVERRIDE; + virtual void Bind(HBasicBlock* block) OVERRIDE; virtual void Move(HInstruction* instruction, Location location, HInstruction* move_for) OVERRIDE; - virtual void SaveCoreRegister(Location stack_location, uint32_t reg_id) OVERRIDE; - virtual void RestoreCoreRegister(Location stack_location, uint32_t reg_id) OVERRIDE; + virtual size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + virtual size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + virtual size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + virtual size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; virtual size_t GetWordSize() const OVERRIDE { return kX86_64WordSize; @@ -188,7 +192,17 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Helper method to move a value between two locations. void Move(Location destination, Location source); + Label* GetLabelOf(HBasicBlock* block) const { + return block_labels_.GetRawStorage() + block->GetBlockId(); + } + + virtual void Initialize() OVERRIDE { + block_labels_.SetSize(GetGraph()->GetBlocks().Size()); + } + private: + // Labels for each block that will be compiled. + GrowableArray<Label> block_labels_; LocationsBuilderX86_64 location_builder_; InstructionCodeGeneratorX86_64 instruction_visitor_; ParallelMoveResolverX86_64 move_resolver_; diff --git a/compiler/optimizing/codegen_test.cc b/compiler/optimizing/codegen_test.cc index 3037f1c2e8..03951e29dd 100644 --- a/compiler/optimizing/codegen_test.cc +++ b/compiler/optimizing/codegen_test.cc @@ -16,8 +16,10 @@ #include <functional> +#include "base/macros.h" #include "builder.h" #include "code_generator_arm.h" +#include "code_generator_arm64.h" #include "code_generator_x86.h" #include "code_generator_x86_64.h" #include "common_compiler_test.h" @@ -93,6 +95,12 @@ static void RunCodeBaseline(HGraph* graph, bool has_result, int32_t expected) { if (kRuntimeISA == kX86_64) { Run(allocator, codegenX86_64, has_result, expected); } + + arm64::CodeGeneratorARM64 codegenARM64(graph); + codegenARM64.CompileBaseline(&allocator, true); + if (kRuntimeISA == kArm64) { + Run(allocator, codegenARM64, has_result, expected); + } } static void RunCodeOptimized(CodeGenerator* codegen, @@ -134,9 +142,9 @@ static void TestCode(const uint16_t* data, bool has_result = false, int32_t expe HGraphBuilder builder(&arena); const DexFile::CodeItem* item = reinterpret_cast<const DexFile::CodeItem*>(data); HGraph* graph = builder.BuildGraph(*item); + ASSERT_NE(graph, nullptr); // Remove suspend checks, they cannot be executed in this context. RemoveSuspendChecks(graph); - ASSERT_NE(graph, nullptr); RunCodeBaseline(graph, has_result, expected); } @@ -260,6 +268,31 @@ TEST(CodegenTest, ReturnIf2) { TestCode(data, true, 0); } +// Exercise bit-wise (one's complement) not-int instruction. +#define NOT_INT_TEST(TEST_NAME, INPUT, EXPECTED_OUTPUT) \ +TEST(CodegenTest, TEST_NAME) { \ + const int32_t input = INPUT; \ + const uint16_t input_lo = input & 0x0000FFFF; \ + const uint16_t input_hi = input >> 16; \ + const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( \ + Instruction::CONST | 0 << 8, input_lo, input_hi, \ + Instruction::NOT_INT | 1 << 8 | 0 << 12 , \ + Instruction::RETURN | 1 << 8); \ + \ + TestCode(data, true, EXPECTED_OUTPUT); \ +} + +NOT_INT_TEST(ReturnNotIntMinus2, -2, 1) +NOT_INT_TEST(ReturnNotIntMinus1, -1, 0) +NOT_INT_TEST(ReturnNotInt0, 0, -1) +NOT_INT_TEST(ReturnNotInt1, 1, -2) +NOT_INT_TEST(ReturnNotIntINT_MIN, -2147483648, 2147483647) // (2^31) - 1 +NOT_INT_TEST(ReturnNotIntINT_MINPlus1, -2147483647, 2147483646) // (2^31) - 2 +NOT_INT_TEST(ReturnNotIntINT_MAXMinus1, 2147483646, -2147483647) // -(2^31) - 1 +NOT_INT_TEST(ReturnNotIntINT_MAX, 2147483647, -2147483648) // -(2^31) + +#undef NOT_INT_TEST + TEST(CodegenTest, ReturnAdd1) { const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 3 << 12 | 0, @@ -349,4 +382,173 @@ TEST(CodegenTest, NonMaterializedCondition) { RunCodeOptimized(graph, hook_before_codegen, true, 0); } +#define MUL_TEST(TYPE, TEST_NAME) \ + TEST(CodegenTest, Return ## TEST_NAME) { \ + const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( \ + Instruction::CONST_4 | 3 << 12 | 0, \ + Instruction::CONST_4 | 4 << 12 | 1 << 8, \ + Instruction::MUL_ ## TYPE, 1 << 8 | 0, \ + Instruction::RETURN); \ + \ + TestCode(data, true, 12); \ + } \ + \ + TEST(CodegenTest, Return ## TEST_NAME ## 2addr) { \ + const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( \ + Instruction::CONST_4 | 3 << 12 | 0, \ + Instruction::CONST_4 | 4 << 12 | 1 << 8, \ + Instruction::MUL_ ## TYPE ## _2ADDR | 1 << 12, \ + Instruction::RETURN); \ + \ + TestCode(data, true, 12); \ + } + +#if !defined(__aarch64__) +MUL_TEST(INT, MulInt); +MUL_TEST(LONG, MulLong); +#endif + +#if defined(__aarch64__) +TEST(CodegenTest, DISABLED_ReturnMulIntLit8) { +#else +TEST(CodegenTest, ReturnMulIntLit8) { +#endif + const uint16_t data[] = ONE_REGISTER_CODE_ITEM( + Instruction::CONST_4 | 4 << 12 | 0 << 8, + Instruction::MUL_INT_LIT8, 3 << 8 | 0, + Instruction::RETURN); + + TestCode(data, true, 12); +} + +#if defined(__aarch64__) +TEST(CodegenTest, DISABLED_ReturnMulIntLit16) { +#else +TEST(CodegenTest, ReturnMulIntLit16) { +#endif + const uint16_t data[] = ONE_REGISTER_CODE_ITEM( + Instruction::CONST_4 | 4 << 12 | 0 << 8, + Instruction::MUL_INT_LIT16, 3, + Instruction::RETURN); + + TestCode(data, true, 12); +} + +TEST(CodegenTest, MaterializedCondition1) { + // Check that condition are materialized correctly. A materialized condition + // should yield `1` if it evaluated to true, and `0` otherwise. + // We force the materialization of comparisons for different combinations of + // inputs and check the results. + + int lhs[] = {1, 2, -1, 2, 0xabc}; + int rhs[] = {2, 1, 2, -1, 0xabc}; + + for (size_t i = 0; i < arraysize(lhs); i++) { + ArenaPool pool; + ArenaAllocator allocator(&pool); + HGraph* graph = new (&allocator) HGraph(&allocator); + + HBasicBlock* entry_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(entry_block); + graph->SetEntryBlock(entry_block); + entry_block->AddInstruction(new (&allocator) HGoto()); + HBasicBlock* code_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(code_block); + HBasicBlock* exit_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(exit_block); + exit_block->AddInstruction(new (&allocator) HExit()); + + entry_block->AddSuccessor(code_block); + code_block->AddSuccessor(exit_block); + graph->SetExitBlock(exit_block); + + HIntConstant cst_lhs(lhs[i]); + code_block->AddInstruction(&cst_lhs); + HIntConstant cst_rhs(rhs[i]); + code_block->AddInstruction(&cst_rhs); + HLessThan cmp_lt(&cst_lhs, &cst_rhs); + code_block->AddInstruction(&cmp_lt); + HReturn ret(&cmp_lt); + code_block->AddInstruction(&ret); + + auto hook_before_codegen = [](HGraph* graph) { + HBasicBlock* block = graph->GetEntryBlock()->GetSuccessors().Get(0); + HParallelMove* move = new (graph->GetArena()) HParallelMove(graph->GetArena()); + block->InsertInstructionBefore(move, block->GetLastInstruction()); + }; + + RunCodeOptimized(graph, hook_before_codegen, true, lhs[i] < rhs[i]); + } +} + +TEST(CodegenTest, MaterializedCondition2) { + // Check that HIf correctly interprets a materialized condition. + // We force the materialization of comparisons for different combinations of + // inputs. An HIf takes the materialized combination as input and returns a + // value that we verify. + + int lhs[] = {1, 2, -1, 2, 0xabc}; + int rhs[] = {2, 1, 2, -1, 0xabc}; + + + for (size_t i = 0; i < arraysize(lhs); i++) { + ArenaPool pool; + ArenaAllocator allocator(&pool); + HGraph* graph = new (&allocator) HGraph(&allocator); + + HBasicBlock* entry_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(entry_block); + graph->SetEntryBlock(entry_block); + entry_block->AddInstruction(new (&allocator) HGoto()); + + HBasicBlock* if_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(if_block); + HBasicBlock* if_true_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(if_true_block); + HBasicBlock* if_false_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(if_false_block); + HBasicBlock* exit_block = new (&allocator) HBasicBlock(graph); + graph->AddBlock(exit_block); + exit_block->AddInstruction(new (&allocator) HExit()); + + graph->SetEntryBlock(entry_block); + entry_block->AddSuccessor(if_block); + if_block->AddSuccessor(if_true_block); + if_block->AddSuccessor(if_false_block); + if_true_block->AddSuccessor(exit_block); + if_false_block->AddSuccessor(exit_block); + graph->SetExitBlock(exit_block); + + HIntConstant cst_lhs(lhs[i]); + if_block->AddInstruction(&cst_lhs); + HIntConstant cst_rhs(rhs[i]); + if_block->AddInstruction(&cst_rhs); + HLessThan cmp_lt(&cst_lhs, &cst_rhs); + if_block->AddInstruction(&cmp_lt); + // We insert a temporary to separate the HIf from the HLessThan and force + // the materialization of the condition. + HTemporary force_materialization(0); + if_block->AddInstruction(&force_materialization); + HIf if_lt(&cmp_lt); + if_block->AddInstruction(&if_lt); + + HIntConstant cst_lt(1); + if_true_block->AddInstruction(&cst_lt); + HReturn ret_lt(&cst_lt); + if_true_block->AddInstruction(&ret_lt); + HIntConstant cst_ge(0); + if_false_block->AddInstruction(&cst_ge); + HReturn ret_ge(&cst_ge); + if_false_block->AddInstruction(&ret_ge); + + auto hook_before_codegen = [](HGraph* graph) { + HBasicBlock* block = graph->GetEntryBlock()->GetSuccessors().Get(0); + HParallelMove* move = new (graph->GetArena()) HParallelMove(graph->GetArena()); + block->InsertInstructionBefore(move, block->GetLastInstruction()); + }; + + RunCodeOptimized(graph, hook_before_codegen, true, lhs[i] < rhs[i]); + } +} + } // namespace art diff --git a/compiler/optimizing/constant_propagation.cc b/compiler/optimizing/constant_folding.cc index d675164fa4..10a7e46299 100644 --- a/compiler/optimizing/constant_propagation.cc +++ b/compiler/optimizing/constant_folding.cc @@ -14,11 +14,11 @@ * limitations under the License. */ -#include "constant_propagation.h" +#include "constant_folding.h" namespace art { -void ConstantPropagation::Run() { +void HConstantFolding::Run() { // Process basic blocks in reverse post-order in the dominator tree, // so that an instruction turned into a constant, used as input of // another instruction, may possibly be used to turn that second @@ -31,11 +31,19 @@ void ConstantPropagation::Run() { for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { HInstruction* inst = it.Current(); - // Constant folding: replace `c <- a op b' with a compile-time - // evaluation of `a op b' if `a' and `b' are constant. if (inst->IsBinaryOperation()) { + // Constant folding: replace `op(a, b)' with a constant at + // compile time if `a' and `b' are both constants. HConstant* constant = - inst->AsBinaryOperation()->TryStaticEvaluation(graph_->GetArena()); + inst->AsBinaryOperation()->TryStaticEvaluation(); + if (constant != nullptr) { + inst->GetBlock()->ReplaceAndRemoveInstructionWith(inst, constant); + } + } else if (inst->IsUnaryOperation()) { + // Constant folding: replace `op(a)' with a constant at compile + // time if `a' is a constant. + HConstant* constant = + inst->AsUnaryOperation()->TryStaticEvaluation(); if (constant != nullptr) { inst->GetBlock()->ReplaceAndRemoveInstructionWith(inst, constant); } diff --git a/compiler/optimizing/constant_folding.h b/compiler/optimizing/constant_folding.h new file mode 100644 index 0000000000..d2acfa6973 --- /dev/null +++ b/compiler/optimizing/constant_folding.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_CONSTANT_FOLDING_H_ +#define ART_COMPILER_OPTIMIZING_CONSTANT_FOLDING_H_ + +#include "nodes.h" +#include "optimization.h" + +namespace art { + +/** + * Optimization pass performing a simple constant-expression + * evaluation on the SSA form. + * + * This class is named art::HConstantFolding to avoid name + * clashes with the art::ConstantPropagation class defined in + * compiler/dex/post_opt_passes.h. + */ +class HConstantFolding : public HOptimization { + public: + HConstantFolding(HGraph* graph, const HGraphVisualizer& visualizer) + : HOptimization(graph, true, kConstantFoldingPassName, visualizer) {} + + virtual void Run() OVERRIDE; + + static constexpr const char* kConstantFoldingPassName = "constant_folding"; + + private: + DISALLOW_COPY_AND_ASSIGN(HConstantFolding); +}; + +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_CONSTANT_FOLDING_H_ diff --git a/compiler/optimizing/constant_propagation_test.cc b/compiler/optimizing/constant_folding_test.cc index 342777a49c..09bf2c8d7d 100644 --- a/compiler/optimizing/constant_propagation_test.cc +++ b/compiler/optimizing/constant_folding_test.cc @@ -16,11 +16,12 @@ #include <functional> -#include "constant_propagation.h" +#include "code_generator_x86.h" +#include "constant_folding.h" #include "dead_code_elimination.h" -#include "pretty_printer.h" #include "graph_checker.h" #include "optimizing_unit_test.h" +#include "pretty_printer.h" #include "gtest/gtest.h" @@ -28,9 +29,9 @@ namespace art { static void TestCode(const uint16_t* data, const std::string& expected_before, - const std::string& expected_after_cp, + const std::string& expected_after_cf, const std::string& expected_after_dce, - std::function<void(HGraph*)> check_after_cp, + std::function<void(HGraph*)> check_after_cf, Primitive::Type return_type = Primitive::kPrimInt) { ArenaPool pool; ArenaAllocator allocator(&pool); @@ -45,29 +46,87 @@ static void TestCode(const uint16_t* data, std::string actual_before = printer_before.str(); ASSERT_EQ(expected_before, actual_before); - ConstantPropagation(graph).Run(); + x86::CodeGeneratorX86 codegen(graph); + HGraphVisualizer visualizer(nullptr, graph, codegen, ""); + HConstantFolding(graph, visualizer).Run(); + SSAChecker ssa_checker(&allocator, graph); + ssa_checker.Run(); + ASSERT_TRUE(ssa_checker.IsValid()); - StringPrettyPrinter printer_after_cp(graph); - printer_after_cp.VisitInsertionOrder(); - std::string actual_after_cp = printer_after_cp.str(); - ASSERT_EQ(expected_after_cp, actual_after_cp); + StringPrettyPrinter printer_after_cf(graph); + printer_after_cf.VisitInsertionOrder(); + std::string actual_after_cf = printer_after_cf.str(); + ASSERT_EQ(expected_after_cf, actual_after_cf); - check_after_cp(graph); + check_after_cf(graph); - DeadCodeElimination(graph).Run(); + HDeadCodeElimination(graph, visualizer).Run(); + ssa_checker.Run(); + ASSERT_TRUE(ssa_checker.IsValid()); StringPrettyPrinter printer_after_dce(graph); printer_after_dce.VisitInsertionOrder(); std::string actual_after_dce = printer_after_dce.str(); ASSERT_EQ(expected_after_dce, actual_after_dce); - - SSAChecker ssa_checker(&allocator, graph); - ssa_checker.VisitInsertionOrder(); - ASSERT_TRUE(ssa_checker.IsValid()); } /** + * Tiny three-register program exercising int constant folding on negation. + * + * 16-bit + * offset + * ------ + * v0 <- 1 0. const/4 v0, #+1 + * v1 <- -v0 1. neg-int v0, v1 + * return v1 2. return v1 + */ +TEST(ConstantFolding, IntConstantFoldingNegation) { + const uint16_t data[] = TWO_REGISTERS_CODE_ITEM( + Instruction::CONST_4 | 0 << 8 | 1 << 12, + Instruction::NEG_INT | 1 << 8 | 0 << 12, + Instruction::RETURN | 1 << 8); + + std::string expected_before = + "BasicBlock 0, succ: 1\n" + " 2: IntConstant [5]\n" + " 10: SuspendCheck\n" + " 11: Goto 1\n" + "BasicBlock 1, pred: 0, succ: 2\n" + " 5: Neg(2) [8]\n" + " 8: Return(5)\n" + "BasicBlock 2, pred: 1\n" + " 9: Exit\n"; + + // Expected difference after constant folding. + diff_t expected_cf_diff = { + { " 2: IntConstant [5]\n", " 2: IntConstant\n" }, + { " 5: Neg(2) [8]\n", " 12: IntConstant [8]\n" }, + { " 8: Return(5)\n", " 8: Return(12)\n" } + }; + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); + + // Check the value of the computed constant. + auto check_after_cf = [](HGraph* graph) { + HInstruction* inst = graph->GetBlock(1)->GetFirstInstruction(); + ASSERT_TRUE(inst->IsIntConstant()); + ASSERT_EQ(inst->AsIntConstant()->GetValue(), -1); + }; + + // Expected difference after dead code elimination. + diff_t expected_dce_diff = { + { " 2: IntConstant\n", removed }, + }; + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); + + TestCode(data, + expected_before, + expected_after_cf, + expected_after_dce, + check_after_cf); +} + +/** * Tiny three-register program exercising int constant folding on addition. * * 16-bit @@ -78,7 +137,7 @@ static void TestCode(const uint16_t* data, * v2 <- v0 + v1 2. add-int v2, v0, v1 * return v2 4. return v2 */ -TEST(ConstantPropagation, IntConstantFoldingOnAddition1) { +TEST(ConstantFolding, IntConstantFoldingOnAddition1) { const uint16_t data[] = THREE_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 << 8 | 1 << 12, Instruction::CONST_4 | 1 << 8 | 2 << 12, @@ -97,17 +156,17 @@ TEST(ConstantPropagation, IntConstantFoldingOnAddition1) { "BasicBlock 2, pred: 1\n" " 13: Exit\n"; - // Expected difference after constant propagation. - diff_t expected_cp_diff = { + // Expected difference after constant folding. + diff_t expected_cf_diff = { { " 3: IntConstant [9]\n", " 3: IntConstant\n" }, { " 5: IntConstant [9]\n", " 5: IntConstant\n" }, { " 9: Add(3, 5) [12]\n", " 16: IntConstant [12]\n" }, { " 12: Return(9)\n", " 12: Return(16)\n" } }; - std::string expected_after_cp = Patch(expected_before, expected_cp_diff); + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); // Check the value of the computed constant. - auto check_after_cp = [](HGraph* graph) { + auto check_after_cf = [](HGraph* graph) { HInstruction* inst = graph->GetBlock(1)->GetFirstInstruction(); ASSERT_TRUE(inst->IsIntConstant()); ASSERT_EQ(inst->AsIntConstant()->GetValue(), 3); @@ -118,13 +177,13 @@ TEST(ConstantPropagation, IntConstantFoldingOnAddition1) { { " 3: IntConstant\n", removed }, { " 5: IntConstant\n", removed } }; - std::string expected_after_dce = Patch(expected_after_cp, expected_dce_diff); + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); TestCode(data, expected_before, - expected_after_cp, + expected_after_cf, expected_after_dce, - check_after_cp); + check_after_cf); } /** @@ -142,7 +201,7 @@ TEST(ConstantPropagation, IntConstantFoldingOnAddition1) { * v2 <- v0 + v1 6. add-int v2, v0, v1 * return v2 8. return v2 */ -TEST(ConstantPropagation, IntConstantFoldingOnAddition2) { +TEST(ConstantFolding, IntConstantFoldingOnAddition2) { const uint16_t data[] = THREE_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 << 8 | 1 << 12, Instruction::CONST_4 | 1 << 8 | 2 << 12, @@ -169,8 +228,8 @@ TEST(ConstantPropagation, IntConstantFoldingOnAddition2) { "BasicBlock 2, pred: 1\n" " 25: Exit\n"; - // Expected difference after constant propagation. - diff_t expected_cp_diff = { + // Expected difference after constant folding. + diff_t expected_cf_diff = { { " 3: IntConstant [9]\n", " 3: IntConstant\n" }, { " 5: IntConstant [9]\n", " 5: IntConstant\n" }, { " 11: IntConstant [17]\n", " 11: IntConstant\n" }, @@ -180,10 +239,10 @@ TEST(ConstantPropagation, IntConstantFoldingOnAddition2) { { " 21: Add(9, 17) [24]\n", " 30: IntConstant [24]\n" }, { " 24: Return(21)\n", " 24: Return(30)\n" } }; - std::string expected_after_cp = Patch(expected_before, expected_cp_diff); + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); // Check the values of the computed constants. - auto check_after_cp = [](HGraph* graph) { + auto check_after_cf = [](HGraph* graph) { HInstruction* inst1 = graph->GetBlock(1)->GetFirstInstruction(); ASSERT_TRUE(inst1->IsIntConstant()); ASSERT_EQ(inst1->AsIntConstant()->GetValue(), 3); @@ -204,13 +263,13 @@ TEST(ConstantPropagation, IntConstantFoldingOnAddition2) { { " 28: IntConstant\n", removed }, { " 29: IntConstant\n", removed } }; - std::string expected_after_dce = Patch(expected_after_cp, expected_dce_diff); + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); TestCode(data, expected_before, - expected_after_cp, + expected_after_cf, expected_after_dce, - check_after_cp); + check_after_cf); } /** @@ -224,7 +283,7 @@ TEST(ConstantPropagation, IntConstantFoldingOnAddition2) { * v2 <- v0 - v1 2. sub-int v2, v0, v1 * return v2 4. return v2 */ -TEST(ConstantPropagation, IntConstantFoldingOnSubtraction) { +TEST(ConstantFolding, IntConstantFoldingOnSubtraction) { const uint16_t data[] = THREE_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 << 8 | 3 << 12, Instruction::CONST_4 | 1 << 8 | 2 << 12, @@ -243,17 +302,17 @@ TEST(ConstantPropagation, IntConstantFoldingOnSubtraction) { "BasicBlock 2, pred: 1\n" " 13: Exit\n"; - // Expected difference after constant propagation. - diff_t expected_cp_diff = { + // Expected difference after constant folding. + diff_t expected_cf_diff = { { " 3: IntConstant [9]\n", " 3: IntConstant\n" }, { " 5: IntConstant [9]\n", " 5: IntConstant\n" }, { " 9: Sub(3, 5) [12]\n", " 16: IntConstant [12]\n" }, { " 12: Return(9)\n", " 12: Return(16)\n" } }; - std::string expected_after_cp = Patch(expected_before, expected_cp_diff); + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); // Check the value of the computed constant. - auto check_after_cp = [](HGraph* graph) { + auto check_after_cf = [](HGraph* graph) { HInstruction* inst = graph->GetBlock(1)->GetFirstInstruction(); ASSERT_TRUE(inst->IsIntConstant()); ASSERT_EQ(inst->AsIntConstant()->GetValue(), 1); @@ -264,13 +323,13 @@ TEST(ConstantPropagation, IntConstantFoldingOnSubtraction) { { " 3: IntConstant\n", removed }, { " 5: IntConstant\n", removed } }; - std::string expected_after_dce = Patch(expected_after_cp, expected_dce_diff); + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); TestCode(data, expected_before, - expected_after_cp, + expected_after_cf, expected_after_dce, - check_after_cp); + check_after_cf); } #define SIX_REGISTERS_CODE_ITEM(...) \ @@ -289,7 +348,7 @@ TEST(ConstantPropagation, IntConstantFoldingOnSubtraction) { * (v0, v1) + (v1, v2) 4. add-long v4, v0, v2 * return (v4, v5) 6. return-wide v4 */ -TEST(ConstantPropagation, LongConstantFoldingOnAddition) { +TEST(ConstantFolding, LongConstantFoldingOnAddition) { const uint16_t data[] = SIX_REGISTERS_CODE_ITEM( Instruction::CONST_WIDE_16 | 0 << 8, 1, Instruction::CONST_WIDE_16 | 2 << 8, 2, @@ -308,17 +367,17 @@ TEST(ConstantPropagation, LongConstantFoldingOnAddition) { "BasicBlock 2, pred: 1\n" " 16: Exit\n"; - // Expected difference after constant propagation. - diff_t expected_cp_diff = { + // Expected difference after constant folding. + diff_t expected_cf_diff = { { " 6: LongConstant [12]\n", " 6: LongConstant\n" }, { " 8: LongConstant [12]\n", " 8: LongConstant\n" }, { " 12: Add(6, 8) [15]\n", " 19: LongConstant [15]\n" }, { " 15: Return(12)\n", " 15: Return(19)\n" } }; - std::string expected_after_cp = Patch(expected_before, expected_cp_diff); + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); // Check the value of the computed constant. - auto check_after_cp = [](HGraph* graph) { + auto check_after_cf = [](HGraph* graph) { HInstruction* inst = graph->GetBlock(1)->GetFirstInstruction(); ASSERT_TRUE(inst->IsLongConstant()); ASSERT_EQ(inst->AsLongConstant()->GetValue(), 3); @@ -329,13 +388,13 @@ TEST(ConstantPropagation, LongConstantFoldingOnAddition) { { " 6: LongConstant\n", removed }, { " 8: LongConstant\n", removed } }; - std::string expected_after_dce = Patch(expected_after_cp, expected_dce_diff); + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); TestCode(data, expected_before, - expected_after_cp, + expected_after_cf, expected_after_dce, - check_after_cp, + check_after_cf, Primitive::kPrimLong); } @@ -352,7 +411,7 @@ TEST(ConstantPropagation, LongConstantFoldingOnAddition) { * (v0, v1) - (v1, v2) 4. sub-long v4, v0, v2 * return (v4, v5) 6. return-wide v4 */ -TEST(ConstantPropagation, LongConstantFoldingOnSubtraction) { +TEST(ConstantFolding, LongConstantFoldingOnSubtraction) { const uint16_t data[] = SIX_REGISTERS_CODE_ITEM( Instruction::CONST_WIDE_16 | 0 << 8, 3, Instruction::CONST_WIDE_16 | 2 << 8, 2, @@ -371,17 +430,17 @@ TEST(ConstantPropagation, LongConstantFoldingOnSubtraction) { "BasicBlock 2, pred: 1\n" " 16: Exit\n"; - // Expected difference after constant propagation. - diff_t expected_cp_diff = { + // Expected difference after constant folding. + diff_t expected_cf_diff = { { " 6: LongConstant [12]\n", " 6: LongConstant\n" }, { " 8: LongConstant [12]\n", " 8: LongConstant\n" }, { " 12: Sub(6, 8) [15]\n", " 19: LongConstant [15]\n" }, { " 15: Return(12)\n", " 15: Return(19)\n" } }; - std::string expected_after_cp = Patch(expected_before, expected_cp_diff); + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); // Check the value of the computed constant. - auto check_after_cp = [](HGraph* graph) { + auto check_after_cf = [](HGraph* graph) { HInstruction* inst = graph->GetBlock(1)->GetFirstInstruction(); ASSERT_TRUE(inst->IsLongConstant()); ASSERT_EQ(inst->AsLongConstant()->GetValue(), 1); @@ -392,13 +451,13 @@ TEST(ConstantPropagation, LongConstantFoldingOnSubtraction) { { " 6: LongConstant\n", removed }, { " 8: LongConstant\n", removed } }; - std::string expected_after_dce = Patch(expected_after_cp, expected_dce_diff); + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); TestCode(data, expected_before, - expected_after_cp, + expected_after_cf, expected_after_dce, - check_after_cp, + check_after_cf, Primitive::kPrimLong); } @@ -424,7 +483,7 @@ TEST(ConstantPropagation, LongConstantFoldingOnSubtraction) { * L3: v2 <- v1 + 4 11. add-int/lit16 v2, v1, #+4 * return v2 13. return v2 */ -TEST(ConstantPropagation, IntConstantFoldingAndJumps) { +TEST(ConstantFolding, IntConstantFoldingAndJumps) { const uint16_t data[] = THREE_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 0 << 8 | 0 << 12, Instruction::CONST_4 | 1 << 8 | 1 << 12, @@ -462,8 +521,8 @@ TEST(ConstantPropagation, IntConstantFoldingAndJumps) { "BasicBlock 5, pred: 4\n" " 29: Exit\n"; - // Expected difference after constant propagation. - diff_t expected_cp_diff = { + // Expected difference after constant folding. + diff_t expected_cf_diff = { { " 3: IntConstant [9]\n", " 3: IntConstant\n" }, { " 5: IntConstant [9]\n", " 5: IntConstant []\n" }, { " 13: IntConstant [14]\n", " 13: IntConstant\n" }, @@ -475,10 +534,10 @@ TEST(ConstantPropagation, IntConstantFoldingAndJumps) { { " 25: Add(14, 24) [28]\n", " 35: IntConstant [28]\n" }, { " 28: Return(25)\n", " 28: Return(35)\n"} }; - std::string expected_after_cp = Patch(expected_before, expected_cp_diff); + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); // Check the values of the computed constants. - auto check_after_cp = [](HGraph* graph) { + auto check_after_cf = [](HGraph* graph) { HInstruction* inst1 = graph->GetBlock(1)->GetFirstInstruction(); ASSERT_TRUE(inst1->IsIntConstant()); ASSERT_EQ(inst1->AsIntConstant()->GetValue(), 1); @@ -501,13 +560,13 @@ TEST(ConstantPropagation, IntConstantFoldingAndJumps) { { " 24: IntConstant\n", removed }, { " 34: IntConstant\n", removed }, }; - std::string expected_after_dce = Patch(expected_after_cp, expected_dce_diff); + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); TestCode(data, expected_before, - expected_after_cp, + expected_after_cf, expected_after_dce, - check_after_cp); + check_after_cf); } @@ -524,7 +583,7 @@ TEST(ConstantPropagation, IntConstantFoldingAndJumps) { * L1: v2 <- v0 + v1 5. add-int v2, v0, v1 * return-void 7. return */ -TEST(ConstantPropagation, ConstantCondition) { +TEST(ConstantFolding, ConstantCondition) { const uint16_t data[] = THREE_REGISTERS_CODE_ITEM( Instruction::CONST_4 | 1 << 8 | 1 << 12, Instruction::CONST_4 | 0 << 8 | 0 << 12, @@ -553,17 +612,17 @@ TEST(ConstantPropagation, ConstantCondition) { "BasicBlock 5, pred: 1, succ: 3\n" " 21: Goto 3\n"; - // Expected difference after constant propagation. - diff_t expected_cp_diff = { + // Expected difference after constant folding. + diff_t expected_cf_diff = { { " 3: IntConstant [15, 22, 8]\n", " 3: IntConstant [15, 22]\n" }, { " 5: IntConstant [22, 8]\n", " 5: IntConstant [22]\n" }, { " 8: GreaterThanOrEqual(3, 5) [9]\n", " 23: IntConstant [9]\n" }, { " 9: If(8)\n", " 9: If(23)\n" } }; - std::string expected_after_cp = Patch(expected_before, expected_cp_diff); + std::string expected_after_cf = Patch(expected_before, expected_cf_diff); // Check the values of the computed constants. - auto check_after_cp = [](HGraph* graph) { + auto check_after_cf = [](HGraph* graph) { HInstruction* inst = graph->GetBlock(1)->GetFirstInstruction(); ASSERT_TRUE(inst->IsIntConstant()); ASSERT_EQ(inst->AsIntConstant()->GetValue(), 1); @@ -575,13 +634,13 @@ TEST(ConstantPropagation, ConstantCondition) { { " 22: Phi(3, 5) [15]\n", " 22: Phi(3, 5)\n" }, { " 15: Add(22, 3)\n", removed } }; - std::string expected_after_dce = Patch(expected_after_cp, expected_dce_diff); + std::string expected_after_dce = Patch(expected_after_cf, expected_dce_diff); TestCode(data, expected_before, - expected_after_cp, + expected_after_cf, expected_after_dce, - check_after_cp); + check_after_cf); } } // namespace art diff --git a/compiler/optimizing/dead_code_elimination.cc b/compiler/optimizing/dead_code_elimination.cc index fe2adc77d0..fc3dd01ef5 100644 --- a/compiler/optimizing/dead_code_elimination.cc +++ b/compiler/optimizing/dead_code_elimination.cc @@ -20,7 +20,7 @@ namespace art { -void DeadCodeElimination::Run() { +void HDeadCodeElimination::Run() { // Process basic blocks in post-order in the dominator tree, so that // a dead instruction depending on another dead instruction is // removed. @@ -35,7 +35,10 @@ void DeadCodeElimination::Run() { for (i.Advance(); !i.Done(); i.Advance()) { HInstruction* inst = i.Current(); DCHECK(!inst->IsControlFlow()); - if (!inst->HasSideEffects() && !inst->HasUses() && !inst->IsSuspendCheck()) { + if (!inst->HasSideEffects() + && !inst->CanThrow() + && !inst->IsSuspendCheck() + && !inst->HasUses()) { block->RemoveInstruction(inst); } } diff --git a/compiler/optimizing/dead_code_elimination.h b/compiler/optimizing/dead_code_elimination.h index 48739be494..a4446ae04d 100644 --- a/compiler/optimizing/dead_code_elimination.h +++ b/compiler/optimizing/dead_code_elimination.h @@ -18,6 +18,7 @@ #define ART_COMPILER_OPTIMIZING_DEAD_CODE_ELIMINATION_H_ #include "nodes.h" +#include "optimization.h" namespace art { @@ -25,17 +26,18 @@ namespace art { * Optimization pass performing dead code elimination (removal of * unused variables/instructions) on the SSA form. */ -class DeadCodeElimination : public ValueObject { +class HDeadCodeElimination : public HOptimization { public: - explicit DeadCodeElimination(HGraph* graph) - : graph_(graph) {} + HDeadCodeElimination(HGraph* graph, const HGraphVisualizer& visualizer) + : HOptimization(graph, true, kDeadCodeEliminationPassName, visualizer) {} - void Run(); + virtual void Run() OVERRIDE; - private: - HGraph* const graph_; + static constexpr const char* kDeadCodeEliminationPassName = + "dead_code_elimination"; - DISALLOW_COPY_AND_ASSIGN(DeadCodeElimination); + private: + DISALLOW_COPY_AND_ASSIGN(HDeadCodeElimination); }; } // namespace art diff --git a/compiler/optimizing/dead_code_elimination_test.cc b/compiler/optimizing/dead_code_elimination_test.cc index 245bcb21d5..0c6807482a 100644 --- a/compiler/optimizing/dead_code_elimination_test.cc +++ b/compiler/optimizing/dead_code_elimination_test.cc @@ -14,10 +14,11 @@ * limitations under the License. */ +#include "code_generator_x86.h" #include "dead_code_elimination.h" -#include "pretty_printer.h" #include "graph_checker.h" #include "optimizing_unit_test.h" +#include "pretty_printer.h" #include "gtest/gtest.h" @@ -39,16 +40,17 @@ static void TestCode(const uint16_t* data, std::string actual_before = printer_before.str(); ASSERT_EQ(actual_before, expected_before); - DeadCodeElimination(graph).Run(); + x86::CodeGeneratorX86 codegen(graph); + HGraphVisualizer visualizer(nullptr, graph, codegen, ""); + HDeadCodeElimination(graph, visualizer).Run(); + SSAChecker ssa_checker(&allocator, graph); + ssa_checker.Run(); + ASSERT_TRUE(ssa_checker.IsValid()); StringPrettyPrinter printer_after(graph); printer_after.VisitInsertionOrder(); std::string actual_after = printer_after.str(); ASSERT_EQ(actual_after, expected_after); - - SSAChecker ssa_checker(&allocator, graph); - ssa_checker.VisitInsertionOrder(); - ASSERT_TRUE(ssa_checker.IsValid()); } @@ -94,6 +96,7 @@ TEST(DeadCodeElimination, AdditionAndConditionalJump) { "BasicBlock 5, pred: 1, succ: 3\n" " 21: Goto 3\n"; + // Expected difference after dead code elimination. diff_t expected_diff = { { " 3: IntConstant [15, 22, 8]\n", " 3: IntConstant [22, 8]\n" }, { " 22: Phi(3, 5) [15]\n", " 22: Phi(3, 5)\n" }, @@ -164,7 +167,7 @@ TEST(DeadCodeElimination, AdditionsAndInconditionalJumps) { "BasicBlock 5, pred: 4\n" " 28: Exit\n"; - // Expected difference after constant propagation. + // Expected difference after dead code elimination. diff_t expected_diff = { { " 13: IntConstant [14]\n", removed }, { " 24: IntConstant [25]\n", removed }, diff --git a/compiler/optimizing/graph_checker.cc b/compiler/optimizing/graph_checker.cc index 589b44a167..743ffc46bf 100644 --- a/compiler/optimizing/graph_checker.cc +++ b/compiler/optimizing/graph_checker.cc @@ -264,21 +264,38 @@ void SSAChecker::CheckLoop(HBasicBlock* loop_header) { void SSAChecker::VisitInstruction(HInstruction* instruction) { super_type::VisitInstruction(instruction); - // Ensure an instruction dominates all its uses (or in the present - // case, that all uses of an instruction (used as input) are - // dominated by its definition). - for (HInputIterator input_it(instruction); !input_it.Done(); - input_it.Advance()) { - HInstruction* input = input_it.Current(); - if (!input->Dominates(instruction)) { + // Ensure an instruction dominates all its uses. + for (HUseIterator<HInstruction> use_it(instruction->GetUses()); + !use_it.Done(); use_it.Advance()) { + HInstruction* use = use_it.Current()->GetUser(); + if (!use->IsPhi() && !instruction->StrictlyDominates(use)) { std::stringstream error; - error << "Instruction " << input->GetId() - << " in block " << input->GetBlock()->GetBlockId() - << " does not dominate use " << instruction->GetId() - << " in block " << current_block_->GetBlockId() << "."; + error << "Instruction " << instruction->GetId() + << " in block " << current_block_->GetBlockId() + << " does not dominate use " << use->GetId() + << " in block " << use->GetBlock()->GetBlockId() << "."; errors_.Insert(error.str()); } } + + // Ensure an instruction having an environment is dominated by the + // instructions contained in the environment. + HEnvironment* environment = instruction->GetEnvironment(); + if (environment != nullptr) { + for (size_t i = 0, e = environment->Size(); i < e; ++i) { + HInstruction* env_instruction = environment->GetInstructionAt(i); + if (env_instruction != nullptr + && !env_instruction->StrictlyDominates(instruction)) { + std::stringstream error; + error << "Instruction " << env_instruction->GetId() + << " in environment of instruction " << instruction->GetId() + << " from block " << current_block_->GetBlockId() + << " does not dominate instruction " << instruction->GetId() + << "."; + errors_.Insert(error.str()); + } + } + } } void SSAChecker::VisitPhi(HPhi* phi) { diff --git a/compiler/optimizing/graph_checker.h b/compiler/optimizing/graph_checker.h index 34a770b5f3..badf21d946 100644 --- a/compiler/optimizing/graph_checker.h +++ b/compiler/optimizing/graph_checker.h @@ -19,15 +19,22 @@ #include "nodes.h" +#include <ostream> + namespace art { // A control-flow graph visitor performing various checks. class GraphChecker : public HGraphVisitor { public: - GraphChecker(ArenaAllocator* allocator, HGraph* graph) + GraphChecker(ArenaAllocator* allocator, HGraph* graph, + const char* dump_prefix = "art::GraphChecker: ") : HGraphVisitor(graph), allocator_(allocator), - errors_(allocator, 0) {} + errors_(allocator, 0), + dump_prefix_(dump_prefix) {} + + // Check the whole graph (in insertion order). + virtual void Run() { VisitInsertionOrder(); } // Check `block`. virtual void VisitBasicBlock(HBasicBlock* block) OVERRIDE; @@ -45,6 +52,13 @@ class GraphChecker : public HGraphVisitor { return errors_; } + // Print detected errors on output stream `os`. + void Dump(std::ostream& os) const { + for (size_t i = 0, e = errors_.Size(); i < e; ++i) { + os << dump_prefix_ << errors_.Get(i) << std::endl; + } + } + protected: ArenaAllocator* const allocator_; // The block currently visited. @@ -53,6 +67,9 @@ class GraphChecker : public HGraphVisitor { GrowableArray<std::string> errors_; private: + // String displayed before dumped errors. + const char* const dump_prefix_; + DISALLOW_COPY_AND_ASSIGN(GraphChecker); }; @@ -63,7 +80,15 @@ class SSAChecker : public GraphChecker { typedef GraphChecker super_type; SSAChecker(ArenaAllocator* allocator, HGraph* graph) - : GraphChecker(allocator, graph) {} + : GraphChecker(allocator, graph, "art::SSAChecker: ") {} + + // Check the whole graph (in reverse post-order). + virtual void Run() { + // VisitReversePostOrder is used instead of VisitInsertionOrder, + // as the latter might visit dead blocks removed by the dominator + // computation. + VisitReversePostOrder(); + } // Perform SSA form checks on `block`. virtual void VisitBasicBlock(HBasicBlock* block) OVERRIDE; diff --git a/compiler/optimizing/graph_checker_test.cc b/compiler/optimizing/graph_checker_test.cc index ea0692088d..39def82007 100644 --- a/compiler/optimizing/graph_checker_test.cc +++ b/compiler/optimizing/graph_checker_test.cc @@ -51,7 +51,7 @@ static void TestCode(const uint16_t* data) { ASSERT_NE(graph, nullptr); GraphChecker graph_checker(&allocator, graph); - graph_checker.VisitInsertionOrder(); + graph_checker.Run(); ASSERT_TRUE(graph_checker.IsValid()); } @@ -65,7 +65,7 @@ static void TestCodeSSA(const uint16_t* data) { graph->TransformToSSA(); SSAChecker ssa_checker(&allocator, graph); - ssa_checker.VisitInsertionOrder(); + ssa_checker.Run(); ASSERT_TRUE(ssa_checker.IsValid()); } @@ -113,13 +113,13 @@ TEST(GraphChecker, InconsistentPredecessorsAndSuccessors) { HGraph* graph = CreateSimpleCFG(&allocator); GraphChecker graph_checker(&allocator, graph); - graph_checker.VisitInsertionOrder(); + graph_checker.Run(); ASSERT_TRUE(graph_checker.IsValid()); // Remove the entry block from the exit block's predecessors, to create an // inconsistent successor/predecessor relation. graph->GetExitBlock()->RemovePredecessor(graph->GetEntryBlock()); - graph_checker.VisitInsertionOrder(); + graph_checker.Run(); ASSERT_FALSE(graph_checker.IsValid()); } @@ -131,7 +131,7 @@ TEST(GraphChecker, BlockEndingWithNonBranchInstruction) { HGraph* graph = CreateSimpleCFG(&allocator); GraphChecker graph_checker(&allocator, graph); - graph_checker.VisitInsertionOrder(); + graph_checker.Run(); ASSERT_TRUE(graph_checker.IsValid()); // Remove the sole instruction of the exit block (composed of a @@ -141,7 +141,7 @@ TEST(GraphChecker, BlockEndingWithNonBranchInstruction) { HInstruction* last_inst = exit_block->GetLastInstruction(); exit_block->RemoveInstruction(last_inst); - graph_checker.VisitInsertionOrder(); + graph_checker.Run(); ASSERT_FALSE(graph_checker.IsValid()); } diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc index b4eb89d30b..4ed2156241 100644 --- a/compiler/optimizing/graph_visualizer.cc +++ b/compiler/optimizing/graph_visualizer.cc @@ -120,13 +120,11 @@ class HGraphVisualizerPrinter : public HGraphVisitor { output_<< std::endl; } - void DumpLocation(Location location, Primitive::Type type) { + void DumpLocation(Location location) { if (location.IsRegister()) { - if (type == Primitive::kPrimDouble || type == Primitive::kPrimFloat) { - codegen_.DumpFloatingPointRegister(output_, location.reg()); - } else { - codegen_.DumpCoreRegister(output_, location.reg()); - } + codegen_.DumpCoreRegister(output_, location.reg()); + } else if (location.IsFpuRegister()) { + codegen_.DumpFloatingPointRegister(output_, location.reg()); } else if (location.IsConstant()) { output_ << "constant"; HConstant* constant = location.GetConstant(); @@ -150,9 +148,9 @@ class HGraphVisualizerPrinter : public HGraphVisitor { output_ << " ("; for (size_t i = 0, e = instruction->NumMoves(); i < e; ++i) { MoveOperands* move = instruction->MoveOperandsAt(i); - DumpLocation(move->GetSource(), Primitive::kPrimInt); + DumpLocation(move->GetSource()); output_ << " -> "; - DumpLocation(move->GetDestination(), Primitive::kPrimInt); + DumpLocation(move->GetDestination()); if (i + 1 != e) { output_ << ", "; } @@ -183,13 +181,13 @@ class HGraphVisualizerPrinter : public HGraphVisitor { if (locations != nullptr) { output_ << " ( "; for (size_t i = 0; i < instruction->InputCount(); ++i) { - DumpLocation(locations->InAt(i), instruction->InputAt(i)->GetType()); + DumpLocation(locations->InAt(i)); output_ << " "; } output_ << ")"; if (locations->Out().IsValid()) { output_ << " -> "; - DumpLocation(locations->Out(), instruction->GetType()); + DumpLocation(locations->Out()); } } output_ << " (liveness: " << instruction->GetLifetimePosition() << ")"; @@ -309,7 +307,7 @@ HGraphVisualizer::HGraphVisualizer(std::ostream* output, printer.EndTag("compilation"); } -void HGraphVisualizer::DumpGraph(const char* pass_name) { +void HGraphVisualizer::DumpGraph(const char* pass_name) const { if (!is_enabled_) { return; } diff --git a/compiler/optimizing/graph_visualizer.h b/compiler/optimizing/graph_visualizer.h index f17ba3bbac..4d8bec2422 100644 --- a/compiler/optimizing/graph_visualizer.h +++ b/compiler/optimizing/graph_visualizer.h @@ -17,6 +17,8 @@ #ifndef ART_COMPILER_OPTIMIZING_GRAPH_VISUALIZER_H_ #define ART_COMPILER_OPTIMIZING_GRAPH_VISUALIZER_H_ +#include <ostream> + #include "base/value_object.h" namespace art { @@ -61,7 +63,7 @@ class HGraphVisualizer : public ValueObject { * If this visualizer is enabled, emit the compilation information * in `output_`. */ - void DumpGraph(const char* pass_name); + void DumpGraph(const char* pass_name) const; private: std::ostream* const output_; diff --git a/compiler/optimizing/gvn.h b/compiler/optimizing/gvn.h index 41b3ceb509..a98d714476 100644 --- a/compiler/optimizing/gvn.h +++ b/compiler/optimizing/gvn.h @@ -17,7 +17,6 @@ #ifndef ART_COMPILER_OPTIMIZING_GVN_H_ #define ART_COMPILER_OPTIMIZING_GVN_H_ -#include <gtest/gtest.h> #include "nodes.h" namespace art { @@ -221,7 +220,7 @@ class GlobalValueNumberer : public ValueObject { // Mark visisted blocks. Only used for debugging. GrowableArray<bool> visited_; - FRIEND_TEST(GVNTest, LoopSideEffects); + ART_FRIEND_TEST(GVNTest, LoopSideEffects); DISALLOW_COPY_AND_ASSIGN(GlobalValueNumberer); }; diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index 2d9e35c3b6..29eabe7e29 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -50,7 +50,7 @@ void InstructionSimplifier::VisitEqual(HEqual* equal) { // Replace (bool_value == 0) with !bool_value DCHECK_EQ(input2->AsIntConstant()->GetValue(), 0); equal->GetBlock()->ReplaceAndRemoveInstructionWith( - equal, new (GetGraph()->GetArena()) HNot(input1)); + equal, new (GetGraph()->GetArena()) HNot(Primitive::kPrimBoolean, input1)); } } } diff --git a/compiler/optimizing/live_ranges_test.cc b/compiler/optimizing/live_ranges_test.cc index d5f4f902c8..89c949563b 100644 --- a/compiler/optimizing/live_ranges_test.cc +++ b/compiler/optimizing/live_ranges_test.cc @@ -73,7 +73,7 @@ TEST(LiveRangesTest, CFG1) { LiveRange* range = interval->GetFirstRange(); ASSERT_EQ(2u, range->GetStart()); // Last use is the return instruction. - ASSERT_EQ(9u, range->GetEnd()); + ASSERT_EQ(8u, range->GetEnd()); HBasicBlock* block = graph->GetBlocks().Get(1); ASSERT_TRUE(block->GetLastInstruction()->IsReturn()); ASSERT_EQ(8u, block->GetLastInstruction()->GetLifetimePosition()); @@ -119,7 +119,7 @@ TEST(LiveRangesTest, CFG2) { LiveRange* range = interval->GetFirstRange(); ASSERT_EQ(2u, range->GetStart()); // Last use is the return instruction. - ASSERT_EQ(23u, range->GetEnd()); + ASSERT_EQ(22u, range->GetEnd()); HBasicBlock* block = graph->GetBlocks().Get(3); ASSERT_TRUE(block->GetLastInstruction()->IsReturn()); ASSERT_EQ(22u, block->GetLastInstruction()->GetLifetimePosition()); @@ -193,7 +193,7 @@ TEST(LiveRangesTest, CFG3) { range = interval->GetFirstRange(); ASSERT_EQ(22u, liveness.GetInstructionFromSsaIndex(2)->GetLifetimePosition()); ASSERT_EQ(22u, range->GetStart()); - ASSERT_EQ(25u, range->GetEnd()); + ASSERT_EQ(24u, range->GetEnd()); ASSERT_TRUE(range->GetNext() == nullptr); } @@ -263,7 +263,7 @@ TEST(LiveRangesTest, Loop1) { range = interval->GetFirstRange(); // The instruction is live until the return instruction after the loop. ASSERT_EQ(6u, range->GetStart()); - ASSERT_EQ(27u, range->GetEnd()); + ASSERT_EQ(26u, range->GetEnd()); ASSERT_TRUE(range->GetNext() == nullptr); // Test for the phi. @@ -271,7 +271,7 @@ TEST(LiveRangesTest, Loop1) { range = interval->GetFirstRange(); // Instruction is consumed by the if. ASSERT_EQ(14u, range->GetStart()); - ASSERT_EQ(16u, range->GetEnd()); + ASSERT_EQ(17u, range->GetEnd()); ASSERT_TRUE(range->GetNext() == nullptr); } @@ -338,7 +338,7 @@ TEST(LiveRangesTest, Loop2) { range = range->GetNext(); ASSERT_TRUE(range != nullptr); ASSERT_EQ(24u, range->GetStart()); - ASSERT_EQ(27u, range->GetEnd()); + ASSERT_EQ(26u, range->GetEnd()); // Test for the add instruction. HAdd* add = liveness.GetInstructionFromSsaIndex(2)->AsAdd(); @@ -410,7 +410,7 @@ TEST(LiveRangesTest, CFG4) { interval = liveness.GetInstructionFromSsaIndex(1)->GetLiveInterval(); range = interval->GetFirstRange(); ASSERT_EQ(4u, range->GetStart()); - ASSERT_EQ(29u, range->GetEnd()); + ASSERT_EQ(28u, range->GetEnd()); ASSERT_TRUE(range->GetNext() == nullptr); // Test for the first add. diff --git a/compiler/optimizing/locations.cc b/compiler/optimizing/locations.cc index 1637484799..ed5e260a5b 100644 --- a/compiler/optimizing/locations.cc +++ b/compiler/optimizing/locations.cc @@ -25,16 +25,14 @@ LocationSummary::LocationSummary(HInstruction* instruction, CallKind call_kind) temps_(instruction->GetBlock()->GetGraph()->GetArena(), 0), environment_(instruction->GetBlock()->GetGraph()->GetArena(), instruction->EnvironmentSize()), - dies_at_entry_(instruction->GetBlock()->GetGraph()->GetArena(), instruction->InputCount()), + output_overlaps_(true), call_kind_(call_kind), stack_mask_(nullptr), register_mask_(0), live_registers_() { inputs_.SetSize(instruction->InputCount()); - dies_at_entry_.SetSize(instruction->InputCount()); for (size_t i = 0; i < instruction->InputCount(); ++i) { inputs_.Put(i, Location()); - dies_at_entry_.Put(i, false); } environment_.SetSize(instruction->EnvironmentSize()); for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) { diff --git a/compiler/optimizing/locations.h b/compiler/optimizing/locations.h index dcf70f27b0..11bcd78521 100644 --- a/compiler/optimizing/locations.h +++ b/compiler/optimizing/locations.h @@ -34,7 +34,7 @@ class HInstruction; */ class Location : public ValueObject { public: - static constexpr bool kDiesAtEntry = true; + static constexpr bool kNoOutputOverlap = false; enum Kind { kInvalid = 0, @@ -373,8 +373,7 @@ class LocationSummary : public ArenaObject { LocationSummary(HInstruction* instruction, CallKind call_kind = kNoCall); - void SetInAt(uint32_t at, Location location, bool dies_at_entry = false) { - dies_at_entry_.Put(at, dies_at_entry); + void SetInAt(uint32_t at, Location location) { inputs_.Put(at, location); } @@ -386,7 +385,8 @@ class LocationSummary : public ArenaObject { return inputs_.Size(); } - void SetOut(Location location) { + void SetOut(Location location, bool overlaps = true) { + output_overlaps_ = overlaps; output_ = Location(location); } @@ -449,23 +449,30 @@ class LocationSummary : public ArenaObject { return &live_registers_; } - bool InputOverlapsWithOutputOrTemp(uint32_t input, bool is_environment) const { + bool InputOverlapsWithOutputOrTemp(uint32_t input_index, bool is_environment) const { if (is_environment) return true; - Location location = Out(); - if (input == 0 && location.IsUnallocated() && location.GetPolicy() == Location::kSameAsFirstInput) { + if ((input_index == 0) + && output_.IsUnallocated() + && (output_.GetPolicy() == Location::kSameAsFirstInput)) { return false; } - if (dies_at_entry_.Get(input)) { + if (inputs_.Get(input_index).IsRegister() || inputs_.Get(input_index).IsFpuRegister()) { return false; } return true; } + bool OutputOverlapsWithInputs() const { + return output_overlaps_; + } + private: GrowableArray<Location> inputs_; GrowableArray<Location> temps_; GrowableArray<Location> environment_; - GrowableArray<bool> dies_at_entry_; + // Whether the output overlaps with any of the inputs. If it overlaps, then it cannot + // share the same register as the inputs. + bool output_overlaps_; Location output_; const CallKind call_kind_; diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index a058dea6b4..d624ad5e5e 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -363,6 +363,25 @@ void HBasicBlock::AddPhi(HPhi* phi) { Add(&phis_, this, phi); } +void HBasicBlock::InsertPhiAfter(HPhi* phi, HPhi* cursor) { + DCHECK_EQ(phi->GetId(), -1); + DCHECK_NE(cursor->GetId(), -1); + DCHECK_EQ(cursor->GetBlock(), this); + if (cursor->next_ == nullptr) { + cursor->next_ = phi; + phi->previous_ = cursor; + DCHECK(phi->next_ == nullptr); + } else { + phi->next_ = cursor->next_; + phi->previous_ = cursor; + cursor->next_ = phi; + phi->next_->previous_ = phi; + } + phi->SetBlock(this); + phi->SetId(GetGraph()->GetNextInstructionId()); + UpdateInputsUsers(phi); +} + static void Remove(HInstructionList* instruction_list, HBasicBlock* block, HInstruction* instruction) { @@ -472,7 +491,11 @@ bool HInstructionList::FoundBefore(const HInstruction* instruction1, return true; } -bool HInstruction::Dominates(HInstruction* other_instruction) const { +bool HInstruction::StrictlyDominates(HInstruction* other_instruction) const { + if (other_instruction == this) { + // An instruction does not strictly dominate itself. + return false; + } HBasicBlock* block = GetBlock(); HBasicBlock* other_block = other_instruction->GetBlock(); if (block != other_block) { @@ -527,6 +550,12 @@ void HInstruction::ReplaceWith(HInstruction* other) { env_uses_ = nullptr; } +void HInstruction::ReplaceInput(HInstruction* replacement, size_t index) { + InputAt(index)->RemoveUser(this, index); + SetRawInputAt(index, replacement); + replacement->AddUseAt(this, index); +} + size_t HInstruction::EnvironmentSize() const { return HasEnvironment() ? environment_->Size() : 0; } @@ -553,6 +582,12 @@ void HGraphVisitor::VisitInsertionOrder() { } } +void HGraphVisitor::VisitReversePostOrder() { + for (HReversePostOrderIterator it(*graph_); !it.Done(); it.Advance()) { + VisitBasicBlock(it.Current()); + } +} + void HGraphVisitor::VisitBasicBlock(HBasicBlock* block) { for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { it.Current()->Accept(this); @@ -562,15 +597,30 @@ void HGraphVisitor::VisitBasicBlock(HBasicBlock* block) { } } -HConstant* HBinaryOperation::TryStaticEvaluation(ArenaAllocator* allocator) const { +HConstant* HUnaryOperation::TryStaticEvaluation() const { + if (GetInput()->IsIntConstant()) { + int32_t value = Evaluate(GetInput()->AsIntConstant()->GetValue()); + return new(GetBlock()->GetGraph()->GetArena()) HIntConstant(value); + } else if (GetInput()->IsLongConstant()) { + // TODO: Implement static evaluation of long unary operations. + // + // Do not exit with a fatal condition here. Instead, simply + // return `nullptr' to notify the caller that this instruction + // cannot (yet) be statically evaluated. + return nullptr; + } + return nullptr; +} + +HConstant* HBinaryOperation::TryStaticEvaluation() const { if (GetLeft()->IsIntConstant() && GetRight()->IsIntConstant()) { int32_t value = Evaluate(GetLeft()->AsIntConstant()->GetValue(), GetRight()->AsIntConstant()->GetValue()); - return new(allocator) HIntConstant(value); + return new(GetBlock()->GetGraph()->GetArena()) HIntConstant(value); } else if (GetLeft()->IsLongConstant() && GetRight()->IsLongConstant()) { int64_t value = Evaluate(GetLeft()->AsLongConstant()->GetValue(), GetRight()->AsLongConstant()->GetValue()); - return new(allocator) HLongConstant(value); + return new(GetBlock()->GetGraph()->GetArena()) HLongConstant(value); } return nullptr; } diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index 677a4f8591..7adb84008a 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -399,6 +399,7 @@ class HBasicBlock : public ArenaObject { void ReplaceAndRemoveInstructionWith(HInstruction* initial, HInstruction* replacement); void AddPhi(HPhi* phi); + void InsertPhiAfter(HPhi* instruction, HPhi* cursor); void RemovePhi(HPhi* phi); bool IsLoopHeader() const { @@ -484,7 +485,7 @@ class HBasicBlock : public ArenaObject { M(Local, Instruction) \ M(LongConstant, Constant) \ M(NewInstance, Instruction) \ - M(Not, Instruction) \ + M(Not, UnaryOperation) \ M(ParameterValue, Instruction) \ M(ParallelMove, Instruction) \ M(Phi, Instruction) \ @@ -502,11 +503,17 @@ class HBasicBlock : public ArenaObject { M(NullCheck, Instruction) \ M(Temporary, Instruction) \ M(SuspendCheck, Instruction) \ + M(Mul, BinaryOperation) \ + M(Neg, UnaryOperation) \ + M(FloatConstant, Constant) \ + M(DoubleConstant, Constant) \ + M(NewArray, Instruction) \ #define FOR_EACH_INSTRUCTION(M) \ FOR_EACH_CONCRETE_INSTRUCTION(M) \ M(Constant, Instruction) \ - M(BinaryOperation, Instruction) \ + M(UnaryOperation, Instruction) \ + M(BinaryOperation, Instruction) \ M(Invoke, Instruction) #define FORWARD_DECLARATION(type, super) class H##type; @@ -650,6 +657,7 @@ class HInstruction : public ArenaObject { virtual bool NeedsEnvironment() const { return false; } virtual bool IsControlFlow() const { return false; } + virtual bool CanThrow() const { return false; } bool HasSideEffects() const { return side_effects_.HasSideEffects(); } void AddUseAt(HInstruction* user, size_t index) { @@ -682,9 +690,10 @@ class HInstruction : public ArenaObject { return result; } - // Does this instruction dominate `other_instruction`? Aborts if - // this instruction and `other_instruction` are both phis. - bool Dominates(HInstruction* other_instruction) const; + // Does this instruction strictly dominate `other_instruction`? + // Returns false if this instruction and `other_instruction` are the same. + // Aborts if this instruction and `other_instruction` are both phis. + bool StrictlyDominates(HInstruction* other_instruction) const; int GetId() const { return id_; } void SetId(int id) { id_ = id; } @@ -705,6 +714,7 @@ class HInstruction : public ArenaObject { void SetLocations(LocationSummary* locations) { locations_ = locations; } void ReplaceWith(HInstruction* instruction); + void ReplaceInput(HInstruction* replacement, size_t index); bool HasOnlyOneUse() const { return uses_ != nullptr && uses_->GetTail() == nullptr; @@ -990,8 +1000,8 @@ class HExpression : public HTemplateInstruction<N> { virtual Primitive::Type GetType() const { return type_; } - private: - const Primitive::Type type_; + protected: + Primitive::Type type_; }; // Represents dex's RETURN_VOID opcode. A HReturnVoid is a control flow @@ -1083,6 +1093,34 @@ class HIf : public HTemplateInstruction<1> { DISALLOW_COPY_AND_ASSIGN(HIf); }; +class HUnaryOperation : public HExpression<1> { + public: + HUnaryOperation(Primitive::Type result_type, HInstruction* input) + : HExpression(result_type, SideEffects::None()) { + SetRawInputAt(0, input); + } + + HInstruction* GetInput() const { return InputAt(0); } + Primitive::Type GetResultType() const { return GetType(); } + + virtual bool CanBeMoved() const { return true; } + virtual bool InstructionDataEquals(HInstruction* other) const { return true; } + + // Try to statically evaluate `operation` and return a HConstant + // containing the result of this evaluation. If `operation` cannot + // be evaluated as a constant, return nullptr. + HConstant* TryStaticEvaluation() const; + + // Apply this operation to `x`. + virtual int32_t Evaluate(int32_t x) const = 0; + virtual int64_t Evaluate(int64_t x) const = 0; + + DECLARE_INSTRUCTION(UnaryOperation); + + private: + DISALLOW_COPY_AND_ASSIGN(HUnaryOperation); +}; + class HBinaryOperation : public HExpression<2> { public: HBinaryOperation(Primitive::Type result_type, @@ -1101,10 +1139,10 @@ class HBinaryOperation : public HExpression<2> { virtual bool CanBeMoved() const { return true; } virtual bool InstructionDataEquals(HInstruction* other) const { return true; } - // Try to statically evaluate `operation` and return an HConstant + // Try to statically evaluate `operation` and return a HConstant // containing the result of this evaluation. If `operation` cannot // be evaluated as a constant, return nullptr. - HConstant* TryStaticEvaluation(ArenaAllocator* allocator) const; + HConstant* TryStaticEvaluation() const; // Apply this operation to `x` and `y`. virtual int32_t Evaluate(int32_t x, int32_t y) const = 0; @@ -1368,6 +1406,48 @@ class HConstant : public HExpression<0> { DISALLOW_COPY_AND_ASSIGN(HConstant); }; +class HFloatConstant : public HConstant { + public: + explicit HFloatConstant(float value) : HConstant(Primitive::kPrimFloat), value_(value) {} + + float GetValue() const { return value_; } + + virtual bool InstructionDataEquals(HInstruction* other) const { + return bit_cast<float, int32_t>(other->AsFloatConstant()->value_) == + bit_cast<float, int32_t>(value_); + } + + virtual size_t ComputeHashCode() const { return static_cast<size_t>(GetValue()); } + + DECLARE_INSTRUCTION(FloatConstant); + + private: + const float value_; + + DISALLOW_COPY_AND_ASSIGN(HFloatConstant); +}; + +class HDoubleConstant : public HConstant { + public: + explicit HDoubleConstant(double value) : HConstant(Primitive::kPrimDouble), value_(value) {} + + double GetValue() const { return value_; } + + virtual bool InstructionDataEquals(HInstruction* other) const { + return bit_cast<double, int64_t>(other->AsDoubleConstant()->value_) == + bit_cast<double, int64_t>(value_); + } + + virtual size_t ComputeHashCode() const { return static_cast<size_t>(GetValue()); } + + DECLARE_INSTRUCTION(DoubleConstant); + + private: + const double value_; + + DISALLOW_COPY_AND_ASSIGN(HDoubleConstant); +}; + // Constants of the type int. Those can be from Dex instructions, or // synthesized (for example with the if-eqz instruction). class HIntConstant : public HConstant { @@ -1515,6 +1595,44 @@ class HNewInstance : public HExpression<0> { DISALLOW_COPY_AND_ASSIGN(HNewInstance); }; +class HNeg : public HUnaryOperation { + public: + explicit HNeg(Primitive::Type result_type, HInstruction* input) + : HUnaryOperation(result_type, input) {} + + virtual int32_t Evaluate(int32_t x) const OVERRIDE { return -x; } + virtual int64_t Evaluate(int64_t x) const OVERRIDE { return -x; } + + DECLARE_INSTRUCTION(Neg); + + private: + DISALLOW_COPY_AND_ASSIGN(HNeg); +}; + +class HNewArray : public HExpression<1> { + public: + HNewArray(HInstruction* length, uint32_t dex_pc, uint16_t type_index) + : HExpression(Primitive::kPrimNot, SideEffects::None()), + dex_pc_(dex_pc), + type_index_(type_index) { + SetRawInputAt(0, length); + } + + uint32_t GetDexPc() const { return dex_pc_; } + uint16_t GetTypeIndex() const { return type_index_; } + + // Calls runtime so needs an environment. + virtual bool NeedsEnvironment() const { return true; } + + DECLARE_INSTRUCTION(NewArray); + + private: + const uint32_t dex_pc_; + const uint16_t type_index_; + + DISALLOW_COPY_AND_ASSIGN(HNewArray); +}; + class HAdd : public HBinaryOperation { public: HAdd(Primitive::Type result_type, HInstruction* left, HInstruction* right) @@ -1555,6 +1673,22 @@ class HSub : public HBinaryOperation { DISALLOW_COPY_AND_ASSIGN(HSub); }; +class HMul : public HBinaryOperation { + public: + HMul(Primitive::Type result_type, HInstruction* left, HInstruction* right) + : HBinaryOperation(result_type, left, right) {} + + virtual bool IsCommutative() { return true; } + + virtual int32_t Evaluate(int32_t x, int32_t y) const { return x * y; } + virtual int64_t Evaluate(int64_t x, int64_t y) const { return x * y; } + + DECLARE_INSTRUCTION(Mul); + + private: + DISALLOW_COPY_AND_ASSIGN(HMul); +}; + // The value of a parameter in this method. Its location depends on // the calling convention. class HParameterValue : public HExpression<0> { @@ -1574,15 +1708,17 @@ class HParameterValue : public HExpression<0> { DISALLOW_COPY_AND_ASSIGN(HParameterValue); }; -class HNot : public HExpression<1> { +class HNot : public HUnaryOperation { public: - explicit HNot(HInstruction* input) : HExpression(Primitive::kPrimBoolean, SideEffects::None()) { - SetRawInputAt(0, input); - } + explicit HNot(Primitive::Type result_type, HInstruction* input) + : HUnaryOperation(result_type, input) {} virtual bool CanBeMoved() const { return true; } virtual bool InstructionDataEquals(HInstruction* other) const { return true; } + virtual int32_t Evaluate(int32_t x) const OVERRIDE { return ~x; } + virtual int64_t Evaluate(int64_t x) const OVERRIDE { return ~x; } + DECLARE_INSTRUCTION(Not); private: @@ -1642,6 +1778,8 @@ class HNullCheck : public HExpression<1> { virtual bool NeedsEnvironment() const { return true; } + virtual bool CanThrow() const { return true; } + uint32_t GetDexPc() const { return dex_pc_; } DECLARE_INSTRUCTION(NullCheck); @@ -1729,6 +1867,7 @@ class HArrayGet : public HExpression<2> { virtual bool CanBeMoved() const { return true; } virtual bool InstructionDataEquals(HInstruction* other) const { return true; } + void SetType(Primitive::Type type) { type_ = type; } DECLARE_INSTRUCTION(ArrayGet); @@ -1741,11 +1880,11 @@ class HArraySet : public HTemplateInstruction<3> { HArraySet(HInstruction* array, HInstruction* index, HInstruction* value, - Primitive::Type component_type, + Primitive::Type expected_component_type, uint32_t dex_pc) : HTemplateInstruction(SideEffects::ChangesSomething()), dex_pc_(dex_pc), - component_type_(component_type) { + expected_component_type_(expected_component_type) { SetRawInputAt(0, array); SetRawInputAt(1, index); SetRawInputAt(2, value); @@ -1759,13 +1898,24 @@ class HArraySet : public HTemplateInstruction<3> { uint32_t GetDexPc() const { return dex_pc_; } - Primitive::Type GetComponentType() const { return component_type_; } + HInstruction* GetValue() const { return InputAt(2); } + + Primitive::Type GetComponentType() const { + // The Dex format does not type floating point index operations. Since the + // `expected_component_type_` is set during building and can therefore not + // be correct, we also check what is the value type. If it is a floating + // point type, we must use that type. + Primitive::Type value_type = GetValue()->GetType(); + return ((value_type == Primitive::kPrimFloat) || (value_type == Primitive::kPrimDouble)) + ? value_type + : expected_component_type_; + } DECLARE_INSTRUCTION(ArraySet); private: const uint32_t dex_pc_; - const Primitive::Type component_type_; + const Primitive::Type expected_component_type_; DISALLOW_COPY_AND_ASSIGN(HArraySet); }; @@ -1802,6 +1952,8 @@ class HBoundsCheck : public HExpression<2> { virtual bool NeedsEnvironment() const { return true; } + virtual bool CanThrow() const { return true; } + uint32_t GetDexPc() const { return dex_pc_; } DECLARE_INSTRUCTION(BoundsCheck); @@ -1956,8 +2108,12 @@ class HGraphVisitor : public ValueObject { virtual void VisitInstruction(HInstruction* instruction) {} virtual void VisitBasicBlock(HBasicBlock* block); + // Visit the graph following basic block insertion order. void VisitInsertionOrder(); + // Visit the graph following dominator tree reverse post-order. + void VisitReversePostOrder(); + HGraph* GetGraph() const { return graph_; } // Visit functions for instruction classes. @@ -1969,7 +2125,7 @@ class HGraphVisitor : public ValueObject { #undef DECLARE_VISIT_INSTRUCTION private: - HGraph* graph_; + HGraph* const graph_; DISALLOW_COPY_AND_ASSIGN(HGraphVisitor); }; diff --git a/compiler/optimizing/constant_propagation.h b/compiler/optimizing/optimization.cc index 0729881888..ea98186d11 100644 --- a/compiler/optimizing/constant_propagation.h +++ b/compiler/optimizing/optimization.cc @@ -14,30 +14,35 @@ * limitations under the License. */ -#ifndef ART_COMPILER_OPTIMIZING_CONSTANT_PROPAGATION_H_ -#define ART_COMPILER_OPTIMIZING_CONSTANT_PROPAGATION_H_ +#include "optimization.h" -#include "nodes.h" +#include "base/dumpable.h" +#include "graph_checker.h" namespace art { -/** - * Optimization pass performing a simple constant propagation on the - * SSA form. - */ -class ConstantPropagation : public ValueObject { - public: - explicit ConstantPropagation(HGraph* graph) - : graph_(graph) {} - - void Run(); - - private: - HGraph* const graph_; - - DISALLOW_COPY_AND_ASSIGN(ConstantPropagation); -}; +void HOptimization::Execute() { + Run(); + visualizer_.DumpGraph(pass_name_); + Check(); +} + +void HOptimization::Check() { + if (kIsDebugBuild) { + if (is_in_ssa_form_) { + SSAChecker checker(graph_->GetArena(), graph_); + checker.Run(); + if (!checker.IsValid()) { + LOG(FATAL) << Dumpable<SSAChecker>(checker); + } + } else { + GraphChecker checker(graph_->GetArena(), graph_); + checker.Run(); + if (!checker.IsValid()) { + LOG(FATAL) << Dumpable<GraphChecker>(checker); + } + } + } +} } // namespace art - -#endif // ART_COMPILER_OPTIMIZING_CONSTANT_PROPAGATION_H_ diff --git a/compiler/optimizing/optimization.h b/compiler/optimizing/optimization.h new file mode 100644 index 0000000000..59683e2075 --- /dev/null +++ b/compiler/optimizing/optimization.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_OPTIMIZATION_H_ +#define ART_COMPILER_OPTIMIZING_OPTIMIZATION_H_ + +#include "graph_visualizer.h" +#include "nodes.h" + +namespace art { + +/** + * Abstraction to implement an optimization pass. + */ +class HOptimization : public ValueObject { + public: + HOptimization(HGraph* graph, + bool is_in_ssa_form, + const char* pass_name, + const HGraphVisualizer& visualizer) + : graph_(graph), + is_in_ssa_form_(is_in_ssa_form), + pass_name_(pass_name), + visualizer_(visualizer) {} + + virtual ~HOptimization() {} + + // Execute the optimization pass. + void Execute(); + + // Return the name of the pass. + const char* GetPassName() const { return pass_name_; } + + // Peform the analysis itself. + virtual void Run() = 0; + + private: + // Verify the graph; abort if it is not valid. + void Check(); + + protected: + HGraph* const graph_; + + private: + // Does the analyzed graph use the SSA form? + const bool is_in_ssa_form_; + // Optimization pass name. + const char* pass_name_; + // A graph visualiser invoked after the execution of the optimization + // pass if enabled. + const HGraphVisualizer& visualizer_; + + DISALLOW_COPY_AND_ASSIGN(HOptimization); +}; + +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_OPTIMIZATION_H_ diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 3cf5a0b291..80e9cdb16f 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -22,6 +22,8 @@ #include "builder.h" #include "code_generator.h" #include "compiler.h" +#include "constant_folding.h" +#include "dead_code_elimination.h" #include "driver/compiler_driver.h" #include "driver/dex_compilation_unit.h" #include "graph_visualizer.h" @@ -213,7 +215,10 @@ CompiledMethod* OptimizingCompiler::TryCompile(const DexFile::CodeItem* code_ite } // Do not attempt to compile on architectures we do not support. - if (instruction_set != kX86 && instruction_set != kX86_64 && instruction_set != kThumb2) { + if (instruction_set != kArm64 && + instruction_set != kThumb2 && + instruction_set != kX86 && + instruction_set != kX86_64) { return nullptr; } @@ -261,6 +266,9 @@ CompiledMethod* OptimizingCompiler::TryCompile(const DexFile::CodeItem* code_ite visualizer.DumpGraph("ssa"); graph->FindNaturalLoops(); + HDeadCodeElimination(graph, visualizer).Execute(); + HConstantFolding(graph, visualizer).Execute(); + SsaRedundantPhiElimination(graph).Run(); SsaDeadPhiElimination(graph).Run(); InstructionSimplifier(graph).Run(); diff --git a/compiler/optimizing/register_allocator.cc b/compiler/optimizing/register_allocator.cc index 5055a76107..f95c4a47e3 100644 --- a/compiler/optimizing/register_allocator.cc +++ b/compiler/optimizing/register_allocator.cc @@ -16,6 +16,8 @@ #include "register_allocator.h" +#include <sstream> + #include "base/bit_vector-inl.h" #include "code_generator.h" #include "ssa_liveness_analysis.h" @@ -37,18 +39,21 @@ RegisterAllocator::RegisterAllocator(ArenaAllocator* allocator, handled_(allocator, 0), active_(allocator, 0), inactive_(allocator, 0), - physical_register_intervals_(allocator, codegen->GetNumberOfCoreRegisters()), + physical_core_register_intervals_(allocator, codegen->GetNumberOfCoreRegisters()), + physical_fp_register_intervals_(allocator, codegen->GetNumberOfFloatingPointRegisters()), temp_intervals_(allocator, 4), spill_slots_(allocator, kDefaultNumberOfSpillSlots), safepoints_(allocator, 0), processing_core_registers_(false), number_of_registers_(-1), registers_array_(nullptr), - blocked_registers_(codegen->GetBlockedCoreRegisters()), + blocked_core_registers_(codegen->GetBlockedCoreRegisters()), + blocked_fp_registers_(codegen->GetBlockedFloatingPointRegisters()), reserved_out_slots_(0), maximum_number_of_live_registers_(0) { codegen->SetupBlockedRegisters(); - physical_register_intervals_.SetSize(codegen->GetNumberOfCoreRegisters()); + physical_core_register_intervals_.SetSize(codegen->GetNumberOfCoreRegisters()); + physical_fp_register_intervals_.SetSize(codegen->GetNumberOfFloatingPointRegisters()); // Always reserve for the current method and the graph's max out registers. // TODO: compute it instead. reserved_out_slots_ = 1 + codegen->GetGraph()->GetMaximumNumberOfOutVRegs(); @@ -65,8 +70,10 @@ bool RegisterAllocator::CanAllocateRegistersFor(const HGraph& graph, it.Advance()) { HInstruction* current = it.Current(); if (current->GetType() == Primitive::kPrimLong && instruction_set != kX86_64) return false; - if (current->GetType() == Primitive::kPrimFloat) return false; - if (current->GetType() == Primitive::kPrimDouble) return false; + if ((current->GetType() == Primitive::kPrimFloat || current->GetType() == Primitive::kPrimDouble) + && instruction_set != kX86_64) { + return false; + } } } return true; @@ -93,14 +100,22 @@ void RegisterAllocator::AllocateRegisters() { void RegisterAllocator::BlockRegister(Location location, size_t start, - size_t end, - Primitive::Type type) { + size_t end) { int reg = location.reg(); - LiveInterval* interval = physical_register_intervals_.Get(reg); + DCHECK(location.IsRegister() || location.IsFpuRegister()); + LiveInterval* interval = location.IsRegister() + ? physical_core_register_intervals_.Get(reg) + : physical_fp_register_intervals_.Get(reg); + Primitive::Type type = location.IsRegister() + ? Primitive::kPrimInt + : Primitive::kPrimDouble; if (interval == nullptr) { interval = LiveInterval::MakeFixedInterval(allocator_, reg, type); - physical_register_intervals_.Put(reg, interval); - inactive_.Add(interval); + if (location.IsRegister()) { + physical_core_register_intervals_.Put(reg, interval); + } else { + physical_fp_register_intervals_.Put(reg, interval); + } } DCHECK(interval->GetRegister() == reg); interval->AddRange(start, end); @@ -123,8 +138,17 @@ void RegisterAllocator::AllocateRegistersInternal() { registers_array_ = allocator_->AllocArray<size_t>(number_of_registers_); processing_core_registers_ = true; unhandled_ = &unhandled_core_intervals_; + for (size_t i = 0, e = physical_core_register_intervals_.Size(); i < e; ++i) { + LiveInterval* fixed = physical_core_register_intervals_.Get(i); + if (fixed != nullptr) { + inactive_.Add(fixed); + } + } LinearScan(); + size_t saved_maximum_number_of_live_registers = maximum_number_of_live_registers_; + maximum_number_of_live_registers_ = 0; + inactive_.Reset(); active_.Reset(); handled_.Reset(); @@ -133,9 +157,14 @@ void RegisterAllocator::AllocateRegistersInternal() { registers_array_ = allocator_->AllocArray<size_t>(number_of_registers_); processing_core_registers_ = false; unhandled_ = &unhandled_fp_intervals_; - // TODO: Enable FP register allocation. - DCHECK(unhandled_->IsEmpty()); + for (size_t i = 0, e = physical_fp_register_intervals_.Size(); i < e; ++i) { + LiveInterval* fixed = physical_fp_register_intervals_.Get(i); + if (fixed != nullptr) { + inactive_.Add(fixed); + } + } LinearScan(); + maximum_number_of_live_registers_ += saved_maximum_number_of_live_registers; } void RegisterAllocator::ProcessInstruction(HInstruction* instruction) { @@ -148,8 +177,9 @@ void RegisterAllocator::ProcessInstruction(HInstruction* instruction) { for (size_t i = 0; i < locations->GetTempCount(); ++i) { Location temp = locations->GetTemp(i); if (temp.IsRegister()) { - BlockRegister(temp, position, position + 1, Primitive::kPrimInt); + BlockRegister(temp, position, position + 1); } else { + DCHECK(temp.IsUnallocated()); LiveInterval* interval = LiveInterval::MakeTempInterval(allocator_, Primitive::kPrimInt); temp_intervals_.Add(interval); interval->AddRange(position, position + 1); @@ -160,10 +190,6 @@ void RegisterAllocator::ProcessInstruction(HInstruction* instruction) { bool core_register = (instruction->GetType() != Primitive::kPrimDouble) && (instruction->GetType() != Primitive::kPrimFloat); - GrowableArray<LiveInterval*>& unhandled = core_register - ? unhandled_core_intervals_ - : unhandled_fp_intervals_; - if (locations->CanCall()) { if (!instruction->IsSuspendCheck()) { codegen_->MarkNotLeaf(); @@ -180,7 +206,8 @@ void RegisterAllocator::ProcessInstruction(HInstruction* instruction) { // maximum before updating locations. LiveInterval* interval = LiveInterval::MakeSlowPathInterval(allocator_, instruction); interval->AddRange(position, position + 1); - unhandled.Add(interval); + unhandled_core_intervals_.Add(interval); + unhandled_fp_intervals_.Add(interval); } } @@ -189,21 +216,29 @@ void RegisterAllocator::ProcessInstruction(HInstruction* instruction) { for (size_t i = 0; i < codegen_->GetNumberOfCoreRegisters(); ++i) { BlockRegister(Location::RegisterLocation(i), position, - position + 1, - Primitive::kPrimInt); + position + 1); + } + for (size_t i = 0; i < codegen_->GetNumberOfFloatingPointRegisters(); ++i) { + BlockRegister(Location::FpuRegisterLocation(i), + position, + position + 1); } } for (size_t i = 0; i < instruction->InputCount(); ++i) { Location input = locations->InAt(i); - if (input.IsRegister()) { - BlockRegister(input, position, position + 1, instruction->InputAt(i)->GetType()); + if (input.IsRegister() || input.IsFpuRegister()) { + BlockRegister(input, position, position + 1); } } LiveInterval* current = instruction->GetLiveInterval(); if (current == nullptr) return; + GrowableArray<LiveInterval*>& unhandled = core_register + ? unhandled_core_intervals_ + : unhandled_fp_intervals_; + DCHECK(unhandled.IsEmpty() || current->StartsBeforeOrAt(unhandled.Peek())); // Some instructions define their output in fixed register/stack slot. We need // to ensure we know these locations before doing register allocation. For a @@ -213,21 +248,24 @@ void RegisterAllocator::ProcessInstruction(HInstruction* instruction) { // // The backwards walking ensures the ranges are ordered on increasing start positions. Location output = locations->Out(); - if (output.IsRegister()) { + if (output.IsRegister() || output.IsFpuRegister()) { // Shift the interval's start by one to account for the blocked register. current->SetFrom(position + 1); current->SetRegister(output.reg()); - BlockRegister(output, position, position + 1, instruction->GetType()); + BlockRegister(output, position, position + 1); + } else if (!locations->OutputOverlapsWithInputs()) { + // Shift the interval's start by one to not interfere with the inputs. + current->SetFrom(position + 1); } else if (output.IsStackSlot() || output.IsDoubleStackSlot()) { current->SetSpillSlot(output.GetStackIndex()); } // If needed, add interval to the list of unhandled intervals. if (current->HasSpillSlot() || instruction->IsConstant()) { - // Split before first register use. + // Split just before first register use. size_t first_register_use = current->FirstRegisterUse(); if (first_register_use != kNoLifetime) { - LiveInterval* split = Split(current, first_register_use); + LiveInterval* split = Split(current, first_register_use - 1); // Don't add direclty to `unhandled`, it needs to be sorted and the start // of this new interval might be after intervals already in the list. AddSorted(&unhandled, split); @@ -278,10 +316,19 @@ bool RegisterAllocator::ValidateInternal(bool log_fatal_on_failure) const { } } - for (size_t i = 0, e = physical_register_intervals_.Size(); i < e; ++i) { - LiveInterval* fixed = physical_register_intervals_.Get(i); - if (fixed != nullptr && ShouldProcess(processing_core_registers_, fixed)) { - intervals.Add(fixed); + if (processing_core_registers_) { + for (size_t i = 0, e = physical_core_register_intervals_.Size(); i < e; ++i) { + LiveInterval* fixed = physical_core_register_intervals_.Get(i); + if (fixed != nullptr) { + intervals.Add(fixed); + } + } + } else { + for (size_t i = 0, e = physical_fp_register_intervals_.Size(); i < e; ++i) { + LiveInterval* fixed = physical_fp_register_intervals_.Get(i); + if (fixed != nullptr) { + intervals.Add(fixed); + } } } @@ -374,10 +421,10 @@ void RegisterAllocator::DumpInterval(std::ostream& stream, LiveInterval* interva interval->Dump(stream); stream << ": "; if (interval->HasRegister()) { - if (processing_core_registers_) { - codegen_->DumpCoreRegister(stream, interval->GetRegister()); - } else { + if (interval->IsFloatingPoint()) { codegen_->DumpFloatingPointRegister(stream, interval->GetRegister()); + } else { + codegen_->DumpCoreRegister(stream, interval->GetRegister()); } } else { stream << "spilled"; @@ -391,6 +438,7 @@ void RegisterAllocator::LinearScan() { // (1) Remove interval with the lowest start position from unhandled. LiveInterval* current = unhandled_->Pop(); DCHECK(!current->IsFixed() && !current->HasSpillSlot()); + DCHECK(unhandled_->IsEmpty() || unhandled_->Peek()->GetStart() >= current->GetStart()); size_t position = current->GetStart(); // (2) Remove currently active intervals that are dead at this position. @@ -519,10 +567,9 @@ bool RegisterAllocator::TryAllocateFreeReg(LiveInterval* current) { } bool RegisterAllocator::IsBlocked(int reg) const { - // TODO: This only works for core registers and needs to be adjusted for - // floating point registers. - DCHECK(processing_core_registers_); - return blocked_registers_[reg]; + return processing_core_registers_ + ? blocked_core_registers_[reg] + : blocked_fp_registers_[reg]; } // Find the register that is used the last, and spill the interval @@ -591,7 +638,9 @@ bool RegisterAllocator::AllocateBlockedReg(LiveInterval* current) { // If the first use of that instruction is after the last use of the found // register, we split this interval just before its first register use. AllocateSpillSlotFor(current); - LiveInterval* split = Split(current, first_register_use); + LiveInterval* split = Split(current, first_register_use - 1); + DCHECK_NE(current, split) << "There is not enough registers available for " + << split->GetParent()->GetDefinedBy()->DebugName(); AddSorted(unhandled_, split); return false; } else { @@ -635,6 +684,7 @@ bool RegisterAllocator::AllocateBlockedReg(LiveInterval* current) { } void RegisterAllocator::AddSorted(GrowableArray<LiveInterval*>* array, LiveInterval* interval) { + DCHECK(!interval->IsFixed() && !interval->HasSpillSlot()); size_t insert_at = 0; for (size_t i = array->Size(); i > 0; --i) { LiveInterval* current = array->Get(i - 1); @@ -723,17 +773,11 @@ void RegisterAllocator::AllocateSpillSlotFor(LiveInterval* interval) { parent->SetSpillSlot((slot + reserved_out_slots_) * kVRegSize); } -// We create a special marker for inputs moves to differentiate them from -// moves created during resolution. They must be different instructions -// because the input moves work on the assumption that the interval moves -// have been executed. -static constexpr size_t kInputMoveLifetimePosition = 0; -static bool IsInputMove(HInstruction* instruction) { - return instruction->GetLifetimePosition() == kInputMoveLifetimePosition; -} - static bool IsValidDestination(Location destination) { - return destination.IsRegister() || destination.IsStackSlot() || destination.IsDoubleStackSlot(); + return destination.IsRegister() + || destination.IsFpuRegister() + || destination.IsStackSlot() + || destination.IsDoubleStackSlot(); } void RegisterAllocator::AddInputMoveFor(HInstruction* user, @@ -748,14 +792,14 @@ void RegisterAllocator::AddInputMoveFor(HInstruction* user, HParallelMove* move = nullptr; if (previous == nullptr || !previous->IsParallelMove() - || !IsInputMove(previous)) { + || previous->GetLifetimePosition() < user->GetLifetimePosition()) { move = new (allocator_) HParallelMove(allocator_); - move->SetLifetimePosition(kInputMoveLifetimePosition); + move->SetLifetimePosition(user->GetLifetimePosition()); user->GetBlock()->InsertInstructionBefore(move, user); } else { move = previous->AsParallelMove(); } - DCHECK(IsInputMove(move)); + DCHECK_EQ(move->GetLifetimePosition(), user->GetLifetimePosition()); move->AddMove(new (allocator_) MoveOperands(source, destination, nullptr)); } @@ -778,7 +822,7 @@ void RegisterAllocator::InsertParallelMoveAt(size_t position, move = at->GetNext()->AsParallelMove(); // This is a parallel move for connecting siblings in a same block. We need to // differentiate it with moves for connecting blocks, and input moves. - if (move == nullptr || IsInputMove(move) || move->GetLifetimePosition() > position) { + if (move == nullptr || move->GetLifetimePosition() > position) { move = new (allocator_) HParallelMove(allocator_); move->SetLifetimePosition(position); at->GetBlock()->InsertInstructionBefore(move, at->GetNext()); @@ -786,12 +830,6 @@ void RegisterAllocator::InsertParallelMoveAt(size_t position, } else { // Move must happen before the instruction. HInstruction* previous = at->GetPrevious(); - if (previous != nullptr && previous->IsParallelMove() && IsInputMove(previous)) { - // This is a parallel move for connecting siblings in a same block. We need to - // differentiate it with input moves. - at = previous; - previous = previous->GetPrevious(); - } if (previous == nullptr || !previous->IsParallelMove() || previous->GetLifetimePosition() != position) { @@ -889,7 +927,9 @@ void RegisterAllocator::ConnectSiblings(LiveInterval* interval) { if (current->HasSpillSlot() && current->HasRegister()) { // We spill eagerly, so move must be at definition. InsertMoveAfter(interval->GetDefinedBy(), - Location::RegisterLocation(interval->GetRegister()), + interval->IsFloatingPoint() + ? Location::FpuRegisterLocation(interval->GetRegister()) + : Location::RegisterLocation(interval->GetRegister()), interval->NeedsTwoSpillSlots() ? Location::DoubleStackSlot(interval->GetParent()->GetSpillSlot()) : Location::StackSlot(interval->GetParent()->GetSpillSlot())); @@ -947,6 +987,10 @@ void RegisterAllocator::ConnectSiblings(LiveInterval* interval) { } break; } + case Location::kFpuRegister: { + locations->AddLiveRegister(source); + break; + } case Location::kStackSlot: // Fall-through case Location::kDoubleStackSlot: // Fall-through case Location::kConstant: { @@ -1110,6 +1154,7 @@ void RegisterAllocator::Resolve() { current = at; } LocationSummary* locations = at->GetLocations(); + DCHECK(temp->GetType() == Primitive::kPrimInt); locations->SetTempAt( temp_index++, Location::RegisterLocation(temp->GetRegister())); } diff --git a/compiler/optimizing/register_allocator.h b/compiler/optimizing/register_allocator.h index d4c233a7f8..b88153969b 100644 --- a/compiler/optimizing/register_allocator.h +++ b/compiler/optimizing/register_allocator.h @@ -21,8 +21,6 @@ #include "primitive.h" #include "utils/growable_array.h" -#include "gtest/gtest.h" - namespace art { class CodeGenerator; @@ -96,7 +94,7 @@ class RegisterAllocator { bool IsBlocked(int reg) const; // Update the interval for the register in `location` to cover [start, end). - void BlockRegister(Location location, size_t start, size_t end, Primitive::Type type); + void BlockRegister(Location location, size_t start, size_t end); // Allocate a spill slot for the given interval. void AllocateSpillSlotFor(LiveInterval* interval); @@ -158,7 +156,8 @@ class RegisterAllocator { // Fixed intervals for physical registers. Such intervals cover the positions // where an instruction requires a specific register. - GrowableArray<LiveInterval*> physical_register_intervals_; + GrowableArray<LiveInterval*> physical_core_register_intervals_; + GrowableArray<LiveInterval*> physical_fp_register_intervals_; // Intervals for temporaries. Such intervals cover the positions // where an instruction requires a temporary. @@ -181,7 +180,8 @@ class RegisterAllocator { size_t* registers_array_; // Blocked registers, as decided by the code generator. - bool* const blocked_registers_; + bool* const blocked_core_registers_; + bool* const blocked_fp_registers_; // Slots reserved for out arguments. size_t reserved_out_slots_; @@ -189,7 +189,7 @@ class RegisterAllocator { // The maximum live registers at safepoints. size_t maximum_number_of_live_registers_; - FRIEND_TEST(RegisterAllocatorTest, FreeUntil); + ART_FRIEND_TEST(RegisterAllocatorTest, FreeUntil); DISALLOW_COPY_AND_ASSIGN(RegisterAllocator); }; diff --git a/compiler/optimizing/register_allocator_test.cc b/compiler/optimizing/register_allocator_test.cc index 7517a6b003..2d84a9d335 100644 --- a/compiler/optimizing/register_allocator_test.cc +++ b/compiler/optimizing/register_allocator_test.cc @@ -348,14 +348,14 @@ TEST(RegisterAllocatorTest, FirstRegisterUse) { // Split at the next instruction. interval = interval->SplitAt(first_add->GetLifetimePosition() + 2); // The user of the split is the last add. - ASSERT_EQ(interval->FirstRegisterUse(), last_add->GetLifetimePosition() - 1); + ASSERT_EQ(interval->FirstRegisterUse(), last_add->GetLifetimePosition()); // Split before the last add. LiveInterval* new_interval = interval->SplitAt(last_add->GetLifetimePosition() - 1); // Ensure the current interval has no register use... ASSERT_EQ(interval->FirstRegisterUse(), kNoLifetime); // And the new interval has it for the last add. - ASSERT_EQ(new_interval->FirstRegisterUse(), last_add->GetLifetimePosition() - 1); + ASSERT_EQ(new_interval->FirstRegisterUse(), last_add->GetLifetimePosition()); } TEST(RegisterAllocatorTest, DeadPhi) { diff --git a/compiler/optimizing/ssa_builder.cc b/compiler/optimizing/ssa_builder.cc index be2c03957d..a0cc8a94ee 100644 --- a/compiler/optimizing/ssa_builder.cc +++ b/compiler/optimizing/ssa_builder.cc @@ -129,8 +129,112 @@ void SsaBuilder::VisitBasicBlock(HBasicBlock* block) { } } +/** + * Constants in the Dex format are not typed. So the builder types them as + * integers, but when doing the SSA form, we might realize the constant + * is used for floating point operations. We create a floating-point equivalent + * constant to make the operations correctly typed. + */ +static HFloatConstant* GetFloatEquivalent(HIntConstant* constant) { + // We place the floating point constant next to this constant. + HFloatConstant* result = constant->GetNext()->AsFloatConstant(); + if (result == nullptr) { + HGraph* graph = constant->GetBlock()->GetGraph(); + ArenaAllocator* allocator = graph->GetArena(); + result = new (allocator) HFloatConstant(bit_cast<int32_t, float>(constant->GetValue())); + constant->GetBlock()->InsertInstructionBefore(result, constant->GetNext()); + } else { + // If there is already a constant with the expected type, we know it is + // the floating point equivalent of this constant. + DCHECK_EQ((bit_cast<float, int32_t>(result->GetValue())), constant->GetValue()); + } + return result; +} + +/** + * Wide constants in the Dex format are not typed. So the builder types them as + * longs, but when doing the SSA form, we might realize the constant + * is used for floating point operations. We create a floating-point equivalent + * constant to make the operations correctly typed. + */ +static HDoubleConstant* GetDoubleEquivalent(HLongConstant* constant) { + // We place the floating point constant next to this constant. + HDoubleConstant* result = constant->GetNext()->AsDoubleConstant(); + if (result == nullptr) { + HGraph* graph = constant->GetBlock()->GetGraph(); + ArenaAllocator* allocator = graph->GetArena(); + result = new (allocator) HDoubleConstant(bit_cast<int64_t, double>(constant->GetValue())); + constant->GetBlock()->InsertInstructionBefore(result, constant->GetNext()); + } else { + // If there is already a constant with the expected type, we know it is + // the floating point equivalent of this constant. + DCHECK_EQ((bit_cast<double, int64_t>(result->GetValue())), constant->GetValue()); + } + return result; +} + +/** + * Because of Dex format, we might end up having the same phi being + * used for non floating point operations and floating point operations. Because + * we want the graph to be correctly typed (and thereafter avoid moves between + * floating point registers and core registers), we need to create a copy of the + * phi with a floating point type. + */ +static HPhi* GetFloatOrDoubleEquivalentOfPhi(HPhi* phi, Primitive::Type type) { + // We place the floating point phi next to this phi. + HInstruction* next = phi->GetNext(); + if (next == nullptr + || (next->GetType() != Primitive::kPrimDouble && next->GetType() != Primitive::kPrimFloat)) { + ArenaAllocator* allocator = phi->GetBlock()->GetGraph()->GetArena(); + HPhi* new_phi = new (allocator) HPhi(allocator, phi->GetRegNumber(), phi->InputCount(), type); + for (size_t i = 0, e = phi->InputCount(); i < e; ++i) { + // Copy the inputs. Note that the graph may not be correctly typed by doing this copy, + // but the type propagation phase will fix it. + new_phi->SetRawInputAt(i, phi->InputAt(i)); + } + phi->GetBlock()->InsertPhiAfter(new_phi, phi); + return new_phi; + } else { + // If there is already a phi with the expected type, we know it is the floating + // point equivalent of this phi. + DCHECK_EQ(next->AsPhi()->GetRegNumber(), phi->GetRegNumber()); + return next->AsPhi(); + } +} + +HInstruction* SsaBuilder::GetFloatOrDoubleEquivalent(HInstruction* user, + HInstruction* value, + Primitive::Type type) { + if (value->IsArrayGet()) { + // The verifier has checked that values in arrays cannot be used for both + // floating point and non-floating point operations. It is therefore safe to just + // change the type of the operation. + value->AsArrayGet()->SetType(type); + return value; + } else if (value->IsLongConstant()) { + return GetDoubleEquivalent(value->AsLongConstant()); + } else if (value->IsIntConstant()) { + return GetFloatEquivalent(value->AsIntConstant()); + } else if (value->IsPhi()) { + return GetFloatOrDoubleEquivalentOfPhi(value->AsPhi(), type); + } else { + // For other instructions, we assume the verifier has checked that the dex format is correctly + // typed and the value in a dex register will not be used for both floating point and + // non-floating point operations. So the only reason an instruction would want a floating + // point equivalent is for an unused phi that will be removed by the dead phi elimination phase. + DCHECK(user->IsPhi()); + return value; + } +} + void SsaBuilder::VisitLoadLocal(HLoadLocal* load) { - load->ReplaceWith(current_locals_->Get(load->GetLocal()->GetRegNumber())); + HInstruction* value = current_locals_->Get(load->GetLocal()->GetRegNumber()); + if (load->GetType() != value->GetType() + && (load->GetType() == Primitive::kPrimFloat || load->GetType() == Primitive::kPrimDouble)) { + // If the operation requests a specific type, we make sure its input is of that type. + value = GetFloatOrDoubleEquivalent(load, value, load->GetType()); + } + load->ReplaceWith(value); load->GetBlock()->RemoveInstruction(load); } diff --git a/compiler/optimizing/ssa_builder.h b/compiler/optimizing/ssa_builder.h index 9d8c0729ae..24f5ac55f7 100644 --- a/compiler/optimizing/ssa_builder.h +++ b/compiler/optimizing/ssa_builder.h @@ -52,6 +52,10 @@ class SsaBuilder : public HGraphVisitor { void VisitStoreLocal(HStoreLocal* store); void VisitInstruction(HInstruction* instruction); + static HInstruction* GetFloatOrDoubleEquivalent(HInstruction* user, + HInstruction* instruction, + Primitive::Type type); + private: // Locals for the current block being visited. GrowableArray<HInstruction*>* current_locals_; diff --git a/compiler/optimizing/ssa_liveness_analysis.cc b/compiler/optimizing/ssa_liveness_analysis.cc index f0edc6422c..1e34670d76 100644 --- a/compiler/optimizing/ssa_liveness_analysis.cc +++ b/compiler/optimizing/ssa_liveness_analysis.cc @@ -319,7 +319,7 @@ int LiveInterval::FindFirstRegisterHint(size_t* free_until) const { if (user->IsPhi()) { // If the phi has a register, try to use the same. Location phi_location = user->GetLiveInterval()->ToLocation(); - if (phi_location.IsRegister() && free_until[phi_location.reg()] >= use_position) { + if (SameRegisterKind(phi_location) && free_until[phi_location.reg()] >= use_position) { return phi_location.reg(); } const GrowableArray<HBasicBlock*>& predecessors = user->GetBlock()->GetPredecessors(); @@ -345,7 +345,7 @@ int LiveInterval::FindFirstRegisterHint(size_t* free_until) const { // We use the user's lifetime position - 1 (and not `use_position`) because the // register is blocked at the beginning of the user. size_t position = user->GetLifetimePosition() - 1; - if (expected.IsRegister() && free_until[expected.reg()] >= position) { + if (SameRegisterKind(expected) && free_until[expected.reg()] >= position) { return expected.reg(); } } @@ -368,7 +368,7 @@ int LiveInterval::FindHintAtDefinition() const { // If the input dies at the end of the predecessor, we know its register can // be reused. Location input_location = input_interval.ToLocation(); - if (input_location.IsRegister()) { + if (SameRegisterKind(input_location)) { return input_location.reg(); } } @@ -384,7 +384,7 @@ int LiveInterval::FindHintAtDefinition() const { // If the input dies at the start of this instruction, we know its register can // be reused. Location location = input_interval.ToLocation(); - if (location.IsRegister()) { + if (SameRegisterKind(location)) { return location.reg(); } } @@ -393,13 +393,21 @@ int LiveInterval::FindHintAtDefinition() const { return kNoRegister; } +bool LiveInterval::SameRegisterKind(Location other) const { + return IsFloatingPoint() + ? other.IsFpuRegister() + : other.IsRegister(); +} + bool LiveInterval::NeedsTwoSpillSlots() const { return type_ == Primitive::kPrimLong || type_ == Primitive::kPrimDouble; } Location LiveInterval::ToLocation() const { if (HasRegister()) { - return Location::RegisterLocation(GetRegister()); + return IsFloatingPoint() + ? Location::FpuRegisterLocation(GetRegister()) + : Location::RegisterLocation(GetRegister()); } else { HInstruction* defined_by = GetParent()->GetDefinedBy(); if (defined_by->IsConstant()) { diff --git a/compiler/optimizing/ssa_liveness_analysis.h b/compiler/optimizing/ssa_liveness_analysis.h index e9bd30338d..7dda4f61d5 100644 --- a/compiler/optimizing/ssa_liveness_analysis.h +++ b/compiler/optimizing/ssa_liveness_analysis.h @@ -32,6 +32,7 @@ class BlockInfo : public ArenaObject { live_in_(allocator, number_of_ssa_values, false), live_out_(allocator, number_of_ssa_values, false), kill_(allocator, number_of_ssa_values, false) { + UNUSED(block_); live_in_.ClearAllBits(); live_out_.ClearAllBits(); kill_.ClearAllBits(); @@ -188,10 +189,14 @@ class LiveInterval : public ArenaObject { && (first_use_->GetPosition() < position)) { // The user uses the instruction multiple times, and one use dies before the other. // We update the use list so that the latter is first. + UsePosition* cursor = first_use_; + while ((cursor->GetNext() != nullptr) && (cursor->GetNext()->GetPosition() < position)) { + cursor = cursor->GetNext(); + } DCHECK(first_use_->GetPosition() + 1 == position); UsePosition* new_use = new (allocator_) UsePosition( - instruction, input_index, is_environment, position, first_use_->GetNext()); - first_use_->SetNext(new_use); + instruction, input_index, is_environment, position, cursor->GetNext()); + cursor->SetNext(new_use); if (first_range_->GetEnd() == first_use_->GetPosition()) { first_range_->end_ = position; } @@ -354,6 +359,10 @@ class LiveInterval : public ArenaObject { || (location.GetPolicy() == Location::kSameAsFirstInput && locations->InAt(0).GetPolicy() == Location::kRequiresRegister)) { return position; + } else if ((location.GetPolicy() == Location::kRequiresFpuRegister) + || (location.GetPolicy() == Location::kSameAsFirstInput + && locations->InAt(0).GetPolicy() == Location::kRequiresFpuRegister)) { + return position; } } } @@ -362,12 +371,12 @@ class LiveInterval : public ArenaObject { size_t end = GetEnd(); while (use != nullptr && use->GetPosition() <= end) { size_t use_position = use->GetPosition(); - if (use_position >= position && !use->GetIsEnvironment()) { + if (use_position > position && !use->GetIsEnvironment()) { Location location = use->GetUser()->GetLocations()->InAt(use->GetInputIndex()); - if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) { - // Return the lifetime just before the user, so that the interval has a register - // when entering the user. - return use->GetUser()->GetLifetimePosition() - 1; + if (location.IsUnallocated() + && (location.GetPolicy() == Location::kRequiresRegister + || location.GetPolicy() == Location::kRequiresFpuRegister)) { + return use_position; } } use = use->GetNext(); @@ -498,6 +507,10 @@ class LiveInterval : public ArenaObject { // slots for spilling. bool NeedsTwoSpillSlots() const; + bool IsFloatingPoint() const { + return type_ == Primitive::kPrimFloat || type_ == Primitive::kPrimDouble; + } + // Converts the location of the interval to a `Location` object. Location ToLocation() const; @@ -509,6 +522,9 @@ class LiveInterval : public ArenaObject { bool IsTemp() const { return is_temp_; } + // Returns whether `other` and `this` share the same kind of register. + bool SameRegisterKind(Location other) const; + private: ArenaAllocator* const allocator_; diff --git a/compiler/optimizing/ssa_phi_elimination.cc b/compiler/optimizing/ssa_phi_elimination.cc index e02a182ec8..4eda0f3757 100644 --- a/compiler/optimizing/ssa_phi_elimination.cc +++ b/compiler/optimizing/ssa_phi_elimination.cc @@ -24,18 +24,13 @@ void SsaDeadPhiElimination::Run() { HBasicBlock* block = it.Current(); for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { HPhi* phi = it.Current()->AsPhi(); - if (phi->HasEnvironmentUses()) { - // TODO: Do we want to keep that phi alive? - worklist_.Add(phi); - phi->SetLive(); - continue; - } for (HUseIterator<HInstruction> it(phi->GetUses()); !it.Done(); it.Advance()) { HUseListNode<HInstruction>* current = it.Current(); HInstruction* user = current->GetUser(); if (!user->IsPhi()) { worklist_.Add(phi); phi->SetLive(); + break; } else { phi->SetDead(); } @@ -76,6 +71,14 @@ void SsaDeadPhiElimination::Run() { current->RemoveUser(user, user_node->GetIndex()); } } + if (current->HasEnvironmentUses()) { + for (HUseIterator<HEnvironment> it(current->GetEnvUses()); !it.Done(); it.Advance()) { + HUseListNode<HEnvironment>* user_node = it.Current(); + HEnvironment* user = user_node->GetUser(); + user->SetRawEnvAt(user_node->GetIndex(), nullptr); + current->RemoveEnvironmentUser(user, user_node->GetIndex()); + } + } block->RemovePhi(current->AsPhi()); } current = next; diff --git a/compiler/optimizing/ssa_type_propagation.cc b/compiler/optimizing/ssa_type_propagation.cc index a860cb7cfe..3828142ed2 100644 --- a/compiler/optimizing/ssa_type_propagation.cc +++ b/compiler/optimizing/ssa_type_propagation.cc @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "ssa_builder.h" #include "ssa_type_propagation.h" #include "nodes.h" @@ -38,15 +39,31 @@ static Primitive::Type MergeTypes(Primitive::Type existing, Primitive::Type new_ // Re-compute and update the type of the instruction. Returns // whether or not the type was changed. -static bool UpdateType(HPhi* phi) { +bool SsaTypePropagation::UpdateType(HPhi* phi) { Primitive::Type existing = phi->GetType(); - Primitive::Type new_type = Primitive::kPrimVoid; + Primitive::Type new_type = existing; for (size_t i = 0, e = phi->InputCount(); i < e; ++i) { Primitive::Type input_type = phi->InputAt(i)->GetType(); new_type = MergeTypes(new_type, input_type); } phi->SetType(new_type); + + if (new_type == Primitive::kPrimDouble || new_type == Primitive::kPrimFloat) { + // If the phi is of floating point type, we need to update its inputs to that + // type. For inputs that are phis, we need to recompute their types. + for (size_t i = 0, e = phi->InputCount(); i < e; ++i) { + HInstruction* input = phi->InputAt(i); + if (input->GetType() != new_type) { + HInstruction* equivalent = SsaBuilder::GetFloatOrDoubleEquivalent(phi, input, new_type); + phi->ReplaceInput(equivalent, i); + if (equivalent->IsPhi()) { + AddToWorklist(equivalent->AsPhi()); + } + } + } + } + return existing != new_type; } @@ -63,7 +80,12 @@ void SsaTypePropagation::VisitBasicBlock(HBasicBlock* block) { HPhi* phi = it.Current()->AsPhi(); // Set the initial type for the phi. Use the non back edge input for reaching // a fixed point faster. - phi->SetType(phi->InputAt(0)->GetType()); + Primitive::Type phi_type = phi->GetType(); + // We merge with the existing type, that has been set by the SSA builder. + DCHECK(phi_type == Primitive::kPrimVoid + || phi_type == Primitive::kPrimFloat + || phi_type == Primitive::kPrimDouble); + phi->SetType(MergeTypes(phi->InputAt(0)->GetType(), phi->GetType())); AddToWorklist(phi); } } else { diff --git a/compiler/optimizing/ssa_type_propagation.h b/compiler/optimizing/ssa_type_propagation.h index 5f471a9811..f4d3d6344a 100644 --- a/compiler/optimizing/ssa_type_propagation.h +++ b/compiler/optimizing/ssa_type_propagation.h @@ -34,6 +34,7 @@ class SsaTypePropagation : public ValueObject { void ProcessWorklist(); void AddToWorklist(HPhi* phi); void AddDependentInstructionsToWorklist(HPhi* phi); + bool UpdateType(HPhi* phi); HGraph* const graph_; GrowableArray<HPhi*> worklist_; |