blob: 8e64e1819e111cbf0f0ee262215f500179859739 [file] [log] [blame]
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +00001/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator.h"
18
Alex Light50fa9932015-08-10 15:30:07 -070019#ifdef ART_ENABLE_CODEGEN_arm
Scott Wakelingfe885462016-09-22 10:24:38 +010020#include "code_generator_arm_vixl.h"
Alex Light50fa9932015-08-10 15:30:07 -070021#endif
22
23#ifdef ART_ENABLE_CODEGEN_arm64
Alexandre Rames5319def2014-10-23 10:03:10 +010024#include "code_generator_arm64.h"
Alex Light50fa9932015-08-10 15:30:07 -070025#endif
26
27#ifdef ART_ENABLE_CODEGEN_x86
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +000028#include "code_generator_x86.h"
Alex Light50fa9932015-08-10 15:30:07 -070029#endif
30
31#ifdef ART_ENABLE_CODEGEN_x86_64
Nicolas Geoffray9cf35522014-06-09 18:40:10 +010032#include "code_generator_x86_64.h"
Alex Light50fa9932015-08-10 15:30:07 -070033#endif
34
Vladimir Marko86c87522020-05-11 16:55:55 +010035#include "art_method-inl.h"
Andreas Gampe5678db52017-06-08 14:11:18 -070036#include "base/bit_utils.h"
37#include "base/bit_utils_iterator.h"
Vladimir Marko174b2e22017-10-12 13:34:49 +010038#include "base/casts.h"
David Sehr67bf42e2018-02-26 16:43:04 -080039#include "base/leb128.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080040#include "class_linker.h"
Yevgeny Roubane3ea8382014-08-08 16:29:38 +070041#include "compiled_method.h"
David Sehr312f3b22018-03-19 08:39:26 -070042#include "dex/bytecode_utils.h"
David Sehr9e734c72018-01-04 17:56:19 -080043#include "dex/code_item_accessors-inl.h"
Nicolas Geoffray92cf83e2014-03-18 17:59:20 +000044#include "dex/verified_method.h"
Alexandre Rameseb7b7392015-06-19 14:47:01 +010045#include "graph_visualizer.h"
Vladimir Markoe47f60c2018-02-21 13:43:28 +000046#include "image.h"
47#include "gc/space/image_space.h"
Andreas Gampec15a2f42017-04-21 12:09:39 -070048#include "intern_table.h"
Nicolas Geoffray5bd05a52015-10-13 09:48:30 +010049#include "intrinsics.h"
Nicolas Geoffray19a19cf2014-10-22 16:07:05 +010050#include "mirror/array-inl.h"
51#include "mirror/object_array-inl.h"
52#include "mirror/object_reference.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080053#include "mirror/reference.h"
Vladimir Markodce016e2016-04-28 13:10:02 +010054#include "mirror/string.h"
Alex Light50fa9932015-08-10 15:30:07 -070055#include "parallel_move_resolver.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080056#include "scoped_thread_state_change-inl.h"
Andreas Gampe8cf9cb32017-07-19 09:28:38 -070057#include "ssa_liveness_analysis.h"
David Srbecky71ec1cc2018-05-18 15:57:25 +010058#include "stack_map.h"
Vladimir Marko174b2e22017-10-12 13:34:49 +010059#include "stack_map_stream.h"
Vladimir Marko552a1342017-10-31 10:56:47 +000060#include "string_builder_append.h"
Andreas Gampeb486a982017-06-01 13:45:54 -070061#include "thread-current-inl.h"
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +000062#include "utils/assembler.h"
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +000063
Vladimir Marko0a516052019-10-14 13:00:44 +000064namespace art {
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +000065
Alexandre Rames88c13cd2015-04-14 17:35:39 +010066// Return whether a location is consistent with a type.
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010067static bool CheckType(DataType::Type type, Location location) {
Alexandre Rames88c13cd2015-04-14 17:35:39 +010068 if (location.IsFpuRegister()
69 || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010070 return (type == DataType::Type::kFloat32) || (type == DataType::Type::kFloat64);
Alexandre Rames88c13cd2015-04-14 17:35:39 +010071 } else if (location.IsRegister() ||
72 (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010073 return DataType::IsIntegralType(type) || (type == DataType::Type::kReference);
Alexandre Rames88c13cd2015-04-14 17:35:39 +010074 } else if (location.IsRegisterPair()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010075 return type == DataType::Type::kInt64;
Alexandre Rames88c13cd2015-04-14 17:35:39 +010076 } else if (location.IsFpuRegisterPair()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010077 return type == DataType::Type::kFloat64;
Alexandre Rames88c13cd2015-04-14 17:35:39 +010078 } else if (location.IsStackSlot()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010079 return (DataType::IsIntegralType(type) && type != DataType::Type::kInt64)
80 || (type == DataType::Type::kFloat32)
81 || (type == DataType::Type::kReference);
Alexandre Rames88c13cd2015-04-14 17:35:39 +010082 } else if (location.IsDoubleStackSlot()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010083 return (type == DataType::Type::kInt64) || (type == DataType::Type::kFloat64);
Alexandre Rames88c13cd2015-04-14 17:35:39 +010084 } else if (location.IsConstant()) {
85 if (location.GetConstant()->IsIntConstant()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010086 return DataType::IsIntegralType(type) && (type != DataType::Type::kInt64);
Alexandre Rames88c13cd2015-04-14 17:35:39 +010087 } else if (location.GetConstant()->IsNullConstant()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010088 return type == DataType::Type::kReference;
Alexandre Rames88c13cd2015-04-14 17:35:39 +010089 } else if (location.GetConstant()->IsLongConstant()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010090 return type == DataType::Type::kInt64;
Alexandre Rames88c13cd2015-04-14 17:35:39 +010091 } else if (location.GetConstant()->IsFloatConstant()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010092 return type == DataType::Type::kFloat32;
Alexandre Rames88c13cd2015-04-14 17:35:39 +010093 } else {
94 return location.GetConstant()->IsDoubleConstant()
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010095 && (type == DataType::Type::kFloat64);
Alexandre Rames88c13cd2015-04-14 17:35:39 +010096 }
97 } else {
98 return location.IsInvalid() || (location.GetPolicy() == Location::kAny);
99 }
100}
101
102// Check that a location summary is consistent with an instruction.
103static bool CheckTypeConsistency(HInstruction* instruction) {
104 LocationSummary* locations = instruction->GetLocations();
105 if (locations == nullptr) {
106 return true;
107 }
108
109 if (locations->Out().IsUnallocated()
110 && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) {
111 DCHECK(CheckType(instruction->GetType(), locations->InAt(0)))
112 << instruction->GetType()
113 << " " << locations->InAt(0);
114 } else {
115 DCHECK(CheckType(instruction->GetType(), locations->Out()))
116 << instruction->GetType()
117 << " " << locations->Out();
118 }
119
Vladimir Markoe9004912016-06-16 16:50:52 +0100120 HConstInputsRef inputs = instruction->GetInputs();
Vladimir Marko372f10e2016-05-17 16:30:10 +0100121 for (size_t i = 0; i < inputs.size(); ++i) {
122 DCHECK(CheckType(inputs[i]->GetType(), locations->InAt(i)))
123 << inputs[i]->GetType() << " " << locations->InAt(i);
Alexandre Rames88c13cd2015-04-14 17:35:39 +0100124 }
125
126 HEnvironment* environment = instruction->GetEnvironment();
127 for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) {
128 if (environment->GetInstructionAt(i) != nullptr) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100129 DataType::Type type = environment->GetInstructionAt(i)->GetType();
Nicolas Geoffray0a23d742015-05-07 11:57:35 +0100130 DCHECK(CheckType(type, environment->GetLocationAt(i)))
131 << type << " " << environment->GetLocationAt(i);
Alexandre Rames88c13cd2015-04-14 17:35:39 +0100132 } else {
Nicolas Geoffray0a23d742015-05-07 11:57:35 +0100133 DCHECK(environment->GetLocationAt(i).IsInvalid())
134 << environment->GetLocationAt(i);
Alexandre Rames88c13cd2015-04-14 17:35:39 +0100135 }
136 }
137 return true;
138}
139
Vladimir Marko174b2e22017-10-12 13:34:49 +0100140class CodeGenerator::CodeGenerationData : public DeletableArenaObject<kArenaAllocCodeGenerator> {
141 public:
142 static std::unique_ptr<CodeGenerationData> Create(ArenaStack* arena_stack,
143 InstructionSet instruction_set) {
144 ScopedArenaAllocator allocator(arena_stack);
145 void* memory = allocator.Alloc<CodeGenerationData>(kArenaAllocCodeGenerator);
146 return std::unique_ptr<CodeGenerationData>(
147 ::new (memory) CodeGenerationData(std::move(allocator), instruction_set));
148 }
149
150 ScopedArenaAllocator* GetScopedAllocator() {
151 return &allocator_;
152 }
153
154 void AddSlowPath(SlowPathCode* slow_path) {
155 slow_paths_.emplace_back(std::unique_ptr<SlowPathCode>(slow_path));
156 }
157
158 ArrayRef<const std::unique_ptr<SlowPathCode>> GetSlowPaths() const {
159 return ArrayRef<const std::unique_ptr<SlowPathCode>>(slow_paths_);
160 }
161
162 StackMapStream* GetStackMapStream() { return &stack_map_stream_; }
163
164 void ReserveJitStringRoot(StringReference string_reference, Handle<mirror::String> string) {
165 jit_string_roots_.Overwrite(string_reference,
166 reinterpret_cast64<uint64_t>(string.GetReference()));
167 }
168
169 uint64_t GetJitStringRootIndex(StringReference string_reference) const {
170 return jit_string_roots_.Get(string_reference);
171 }
172
173 size_t GetNumberOfJitStringRoots() const {
174 return jit_string_roots_.size();
175 }
176
177 void ReserveJitClassRoot(TypeReference type_reference, Handle<mirror::Class> klass) {
178 jit_class_roots_.Overwrite(type_reference, reinterpret_cast64<uint64_t>(klass.GetReference()));
179 }
180
181 uint64_t GetJitClassRootIndex(TypeReference type_reference) const {
182 return jit_class_roots_.Get(type_reference);
183 }
184
185 size_t GetNumberOfJitClassRoots() const {
186 return jit_class_roots_.size();
187 }
188
189 size_t GetNumberOfJitRoots() const {
190 return GetNumberOfJitStringRoots() + GetNumberOfJitClassRoots();
191 }
192
Vladimir Markoac3ac682018-09-20 11:01:43 +0100193 void EmitJitRoots(/*out*/std::vector<Handle<mirror::Object>>* roots)
Vladimir Marko174b2e22017-10-12 13:34:49 +0100194 REQUIRES_SHARED(Locks::mutator_lock_);
195
196 private:
197 CodeGenerationData(ScopedArenaAllocator&& allocator, InstructionSet instruction_set)
198 : allocator_(std::move(allocator)),
199 stack_map_stream_(&allocator_, instruction_set),
200 slow_paths_(allocator_.Adapter(kArenaAllocCodeGenerator)),
201 jit_string_roots_(StringReferenceValueComparator(),
202 allocator_.Adapter(kArenaAllocCodeGenerator)),
203 jit_class_roots_(TypeReferenceValueComparator(),
204 allocator_.Adapter(kArenaAllocCodeGenerator)) {
205 slow_paths_.reserve(kDefaultSlowPathsCapacity);
206 }
207
208 static constexpr size_t kDefaultSlowPathsCapacity = 8;
209
210 ScopedArenaAllocator allocator_;
211 StackMapStream stack_map_stream_;
212 ScopedArenaVector<std::unique_ptr<SlowPathCode>> slow_paths_;
213
214 // Maps a StringReference (dex_file, string_index) to the index in the literal table.
215 // Entries are intially added with a pointer in the handle zone, and `EmitJitRoots`
216 // will compute all the indices.
217 ScopedArenaSafeMap<StringReference, uint64_t, StringReferenceValueComparator> jit_string_roots_;
218
219 // Maps a ClassReference (dex_file, type_index) to the index in the literal table.
220 // Entries are intially added with a pointer in the handle zone, and `EmitJitRoots`
221 // will compute all the indices.
222 ScopedArenaSafeMap<TypeReference, uint64_t, TypeReferenceValueComparator> jit_class_roots_;
223};
224
225void CodeGenerator::CodeGenerationData::EmitJitRoots(
Vladimir Markoac3ac682018-09-20 11:01:43 +0100226 /*out*/std::vector<Handle<mirror::Object>>* roots) {
227 DCHECK(roots->empty());
228 roots->reserve(GetNumberOfJitRoots());
Vladimir Marko174b2e22017-10-12 13:34:49 +0100229 ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
230 size_t index = 0;
231 for (auto& entry : jit_string_roots_) {
232 // Update the `roots` with the string, and replace the address temporarily
233 // stored to the index in the table.
234 uint64_t address = entry.second;
Vladimir Markoac3ac682018-09-20 11:01:43 +0100235 roots->emplace_back(reinterpret_cast<StackReference<mirror::Object>*>(address));
236 DCHECK(roots->back() != nullptr);
237 DCHECK(roots->back()->IsString());
Vladimir Marko174b2e22017-10-12 13:34:49 +0100238 entry.second = index;
239 // Ensure the string is strongly interned. This is a requirement on how the JIT
240 // handles strings. b/32995596
Vladimir Markoac3ac682018-09-20 11:01:43 +0100241 class_linker->GetInternTable()->InternStrong(roots->back()->AsString());
Vladimir Marko174b2e22017-10-12 13:34:49 +0100242 ++index;
243 }
244 for (auto& entry : jit_class_roots_) {
245 // Update the `roots` with the class, and replace the address temporarily
246 // stored to the index in the table.
247 uint64_t address = entry.second;
Vladimir Markoac3ac682018-09-20 11:01:43 +0100248 roots->emplace_back(reinterpret_cast<StackReference<mirror::Object>*>(address));
249 DCHECK(roots->back() != nullptr);
250 DCHECK(roots->back()->IsClass());
Vladimir Marko174b2e22017-10-12 13:34:49 +0100251 entry.second = index;
252 ++index;
253 }
254}
255
256ScopedArenaAllocator* CodeGenerator::GetScopedAllocator() {
257 DCHECK(code_generation_data_ != nullptr);
258 return code_generation_data_->GetScopedAllocator();
259}
260
261StackMapStream* CodeGenerator::GetStackMapStream() {
262 DCHECK(code_generation_data_ != nullptr);
263 return code_generation_data_->GetStackMapStream();
264}
265
266void CodeGenerator::ReserveJitStringRoot(StringReference string_reference,
267 Handle<mirror::String> string) {
268 DCHECK(code_generation_data_ != nullptr);
269 code_generation_data_->ReserveJitStringRoot(string_reference, string);
270}
271
272uint64_t CodeGenerator::GetJitStringRootIndex(StringReference string_reference) {
273 DCHECK(code_generation_data_ != nullptr);
274 return code_generation_data_->GetJitStringRootIndex(string_reference);
275}
276
277void CodeGenerator::ReserveJitClassRoot(TypeReference type_reference, Handle<mirror::Class> klass) {
278 DCHECK(code_generation_data_ != nullptr);
279 code_generation_data_->ReserveJitClassRoot(type_reference, klass);
280}
281
282uint64_t CodeGenerator::GetJitClassRootIndex(TypeReference type_reference) {
283 DCHECK(code_generation_data_ != nullptr);
284 return code_generation_data_->GetJitClassRootIndex(type_reference);
285}
286
287void CodeGenerator::EmitJitRootPatches(uint8_t* code ATTRIBUTE_UNUSED,
288 const uint8_t* roots_data ATTRIBUTE_UNUSED) {
289 DCHECK(code_generation_data_ != nullptr);
290 DCHECK_EQ(code_generation_data_->GetNumberOfJitStringRoots(), 0u);
291 DCHECK_EQ(code_generation_data_->GetNumberOfJitClassRoots(), 0u);
292}
293
Vladimir Markodce016e2016-04-28 13:10:02 +0100294uint32_t CodeGenerator::GetArrayLengthOffset(HArrayLength* array_length) {
295 return array_length->IsStringLength()
296 ? mirror::String::CountOffset().Uint32Value()
297 : mirror::Array::LengthOffset().Uint32Value();
298}
299
Vladimir Marko87f3fcb2016-04-28 15:52:11 +0100300uint32_t CodeGenerator::GetArrayDataOffset(HArrayGet* array_get) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100301 DCHECK(array_get->GetType() == DataType::Type::kUint16 || !array_get->IsStringCharAt());
Vladimir Marko87f3fcb2016-04-28 15:52:11 +0100302 return array_get->IsStringCharAt()
303 ? mirror::String::ValueOffset().Uint32Value()
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100304 : mirror::Array::DataOffset(DataType::Size(array_get->GetType())).Uint32Value();
Vladimir Marko87f3fcb2016-04-28 15:52:11 +0100305}
306
Nicolas Geoffraydc23d832015-02-16 11:15:43 +0000307bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const {
Vladimir Markofa6b93c2015-09-15 10:15:55 +0100308 DCHECK_EQ((*block_order_)[current_block_index_], current);
Nicolas Geoffraydc23d832015-02-16 11:15:43 +0000309 return GetNextBlockToEmit() == FirstNonEmptyBlock(next);
310}
311
312HBasicBlock* CodeGenerator::GetNextBlockToEmit() const {
Vladimir Markofa6b93c2015-09-15 10:15:55 +0100313 for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) {
314 HBasicBlock* block = (*block_order_)[i];
David Brazdilfc6a86a2015-06-26 10:33:45 +0000315 if (!block->IsSingleJump()) {
Nicolas Geoffraydc23d832015-02-16 11:15:43 +0000316 return block;
317 }
318 }
319 return nullptr;
320}
321
322HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const {
David Brazdilfc6a86a2015-06-26 10:33:45 +0000323 while (block->IsSingleJump()) {
Vladimir Markoec7802a2015-10-01 20:57:57 +0100324 block = block->GetSuccessors()[0];
Nicolas Geoffraydc23d832015-02-16 11:15:43 +0000325 }
326 return block;
327}
328
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100329class DisassemblyScope {
330 public:
331 DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen)
332 : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) {
333 if (codegen_.GetDisassemblyInformation() != nullptr) {
334 start_offset_ = codegen_.GetAssembler().CodeSize();
335 }
336 }
337
338 ~DisassemblyScope() {
339 // We avoid building this data when we know it will not be used.
340 if (codegen_.GetDisassemblyInformation() != nullptr) {
341 codegen_.GetDisassemblyInformation()->AddInstructionInterval(
342 instruction_, start_offset_, codegen_.GetAssembler().CodeSize());
343 }
344 }
345
346 private:
347 const CodeGenerator& codegen_;
348 HInstruction* instruction_;
349 size_t start_offset_;
350};
351
352
353void CodeGenerator::GenerateSlowPaths() {
Vladimir Marko174b2e22017-10-12 13:34:49 +0100354 DCHECK(code_generation_data_ != nullptr);
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100355 size_t code_start = 0;
Vladimir Marko174b2e22017-10-12 13:34:49 +0100356 for (const std::unique_ptr<SlowPathCode>& slow_path_ptr : code_generation_data_->GetSlowPaths()) {
357 SlowPathCode* slow_path = slow_path_ptr.get();
Vladimir Marko0f7dca42015-11-02 14:36:43 +0000358 current_slow_path_ = slow_path;
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100359 if (disasm_info_ != nullptr) {
360 code_start = GetAssembler()->CodeSize();
361 }
David Srbecky9cd6d372016-02-09 15:24:47 +0000362 // Record the dex pc at start of slow path (required for java line number mapping).
David Srbeckyd28f4a02016-03-14 17:14:24 +0000363 MaybeRecordNativeDebugInfo(slow_path->GetInstruction(), slow_path->GetDexPc(), slow_path);
Vladimir Marko225b6462015-09-28 12:17:40 +0100364 slow_path->EmitNativeCode(this);
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100365 if (disasm_info_ != nullptr) {
Vladimir Marko225b6462015-09-28 12:17:40 +0100366 disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize());
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100367 }
368 }
Vladimir Marko0f7dca42015-11-02 14:36:43 +0000369 current_slow_path_ = nullptr;
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100370}
371
Vladimir Marko174b2e22017-10-12 13:34:49 +0100372void CodeGenerator::InitializeCodeGenerationData() {
373 DCHECK(code_generation_data_ == nullptr);
374 code_generation_data_ = CodeGenerationData::Create(graph_->GetArenaStack(), GetInstructionSet());
375}
376
David Brazdil58282f42016-01-14 12:45:10 +0000377void CodeGenerator::Compile(CodeAllocator* allocator) {
Vladimir Marko174b2e22017-10-12 13:34:49 +0100378 InitializeCodeGenerationData();
379
David Brazdil58282f42016-01-14 12:45:10 +0000380 // The register allocator already called `InitializeCodeGeneration`,
381 // where the frame size has been computed.
382 DCHECK(block_order_ != nullptr);
383 Initialize();
384
Nicolas Geoffray8a16d972014-09-11 10:30:02 +0100385 HGraphVisitor* instruction_visitor = GetInstructionVisitor();
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000386 DCHECK_EQ(current_block_index_, 0u);
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100387
David Srbeckyf6ba5b32018-06-23 22:05:49 +0100388 GetStackMapStream()->BeginMethod(HasEmptyFrame() ? 0 : frame_size_,
389 core_spill_mask_,
390 fpu_spill_mask_,
Nicolas Geoffraya59af8a2019-11-27 17:42:32 +0000391 GetGraph()->GetNumberOfVRegs(),
392 GetGraph()->IsCompilingBaseline());
David Srbeckyf6ba5b32018-06-23 22:05:49 +0100393
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100394 size_t frame_start = GetAssembler()->CodeSize();
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000395 GenerateFrameEntry();
David Srbeckyc6b4dd82015-04-07 20:32:43 +0100396 DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_));
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100397 if (disasm_info_ != nullptr) {
398 disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize());
399 }
400
Vladimir Markofa6b93c2015-09-15 10:15:55 +0100401 for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) {
402 HBasicBlock* block = (*block_order_)[current_block_index_];
Nicolas Geoffraydc23d832015-02-16 11:15:43 +0000403 // Don't generate code for an empty block. Its predecessors will branch to its successor
404 // directly. Also, the label of that block will not be emitted, so this helps catch
405 // errors where we reference that label.
David Brazdilfc6a86a2015-06-26 10:33:45 +0000406 if (block->IsSingleJump()) continue;
Nicolas Geoffray92a73ae2014-10-16 11:12:52 +0100407 Bind(block);
David Srbeckyc7098ff2016-02-09 14:30:11 +0000408 // This ensures that we have correct native line mapping for all native instructions.
409 // It is necessary to make stepping over a statement work. Otherwise, any initial
410 // instructions (e.g. moves) would be assumed to be the start of next statement.
Andreas Gampe3db70682018-12-26 15:12:03 -0800411 MaybeRecordNativeDebugInfo(/* instruction= */ nullptr, block->GetDexPc());
Nicolas Geoffray86dbb9a2014-06-04 11:12:39 +0100412 for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
413 HInstruction* current = it.Current();
David Srbeckyd28f4a02016-03-14 17:14:24 +0000414 if (current->HasEnvironment()) {
415 // Create stackmap for HNativeDebugInfo or any instruction which calls native code.
416 // Note that we need correct mapping for the native PC of the call instruction,
417 // so the runtime's stackmap is not sufficient since it is at PC after the call.
418 MaybeRecordNativeDebugInfo(current, block->GetDexPc());
419 }
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100420 DisassemblyScope disassembly_scope(current, *this);
Alexandre Rames88c13cd2015-04-14 17:35:39 +0100421 DCHECK(CheckTypeConsistency(current));
Nicolas Geoffray86dbb9a2014-06-04 11:12:39 +0100422 current->Accept(instruction_visitor);
423 }
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +0000424 }
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000425
Alexandre Rameseb7b7392015-06-19 14:47:01 +0100426 GenerateSlowPaths();
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000427
David Brazdil77a48ae2015-09-15 12:34:04 +0000428 // Emit catch stack maps at the end of the stack map stream as expected by the
429 // runtime exception handler.
David Brazdil58282f42016-01-14 12:45:10 +0000430 if (graph_->HasTryCatch()) {
David Brazdil77a48ae2015-09-15 12:34:04 +0000431 RecordCatchBlockInfo();
432 }
433
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000434 // Finalize instructions in assember;
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +0000435 Finalize(allocator);
David Srbeckyf6ba5b32018-06-23 22:05:49 +0100436
Nicolas Geoffraybf5f0f32019-03-05 15:41:50 +0000437 GetStackMapStream()->EndMethod();
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +0000438}
439
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +0000440void CodeGenerator::Finalize(CodeAllocator* allocator) {
Nicolas Geoffray86dbb9a2014-06-04 11:12:39 +0100441 size_t code_size = GetAssembler()->CodeSize();
442 uint8_t* buffer = allocator->Allocate(code_size);
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +0000443
Nicolas Geoffray86dbb9a2014-06-04 11:12:39 +0100444 MemoryRegion code(buffer, code_size);
445 GetAssembler()->FinalizeInstructions(code);
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +0000446}
447
Vladimir Markod8dbc8d2017-09-20 13:37:47 +0100448void CodeGenerator::EmitLinkerPatches(
449 ArenaVector<linker::LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) {
Vladimir Marko58155012015-08-19 12:49:41 +0000450 // No linker patches by default.
451}
452
Vladimir Markoca1e0382018-04-11 09:58:41 +0000453bool CodeGenerator::NeedsThunkCode(const linker::LinkerPatch& patch ATTRIBUTE_UNUSED) const {
454 // Code generators that create patches requiring thunk compilation should override this function.
455 return false;
456}
457
458void CodeGenerator::EmitThunkCode(const linker::LinkerPatch& patch ATTRIBUTE_UNUSED,
459 /*out*/ ArenaVector<uint8_t>* code ATTRIBUTE_UNUSED,
460 /*out*/ std::string* debug_name ATTRIBUTE_UNUSED) {
461 // Code generators that create patches requiring thunk compilation should override this function.
462 LOG(FATAL) << "Unexpected call to EmitThunkCode().";
463}
464
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000465void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
Vladimir Marko70e97462016-08-09 11:04:26 +0100466 size_t maximum_safepoint_spill_size,
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000467 size_t number_of_out_slots,
Vladimir Markofa6b93c2015-09-15 10:15:55 +0100468 const ArenaVector<HBasicBlock*>& block_order) {
Nicolas Geoffray4c204ba2015-02-03 15:12:35 +0000469 block_order_ = &block_order;
Vladimir Markofa6b93c2015-09-15 10:15:55 +0100470 DCHECK(!block_order.empty());
471 DCHECK(block_order[0] == GetGraph()->GetEntryBlock());
Nicolas Geoffray4dee6362015-01-23 18:23:14 +0000472 ComputeSpillMask();
Alexandre Rames68bd9b92016-07-15 17:41:13 +0100473 first_register_slot_in_slow_path_ = RoundUp(
474 (number_of_out_slots + number_of_spill_slots) * kVRegSize, GetPreferredSlotsAlignment());
Nicolas Geoffray3bca0df2014-09-19 11:01:00 +0100475
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000476 if (number_of_spill_slots == 0
477 && !HasAllocatedCalleeSaveRegisters()
478 && IsLeafMethod()
479 && !RequiresCurrentMethod()) {
Vladimir Marko70e97462016-08-09 11:04:26 +0100480 DCHECK_EQ(maximum_safepoint_spill_size, 0u);
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000481 SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
482 } else {
483 SetFrameSize(RoundUp(
Alexandre Rames68bd9b92016-07-15 17:41:13 +0100484 first_register_slot_in_slow_path_
Vladimir Marko70e97462016-08-09 11:04:26 +0100485 + maximum_safepoint_spill_size
Mingyao Yang063fc772016-08-02 11:02:54 -0700486 + (GetGraph()->HasShouldDeoptimizeFlag() ? kShouldDeoptimizeFlagSize : 0)
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000487 + FrameEntrySpillSize(),
488 kStackAlignment));
489 }
Nicolas Geoffrayab032bc2014-07-15 12:55:21 +0100490}
491
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +0100492void CodeGenerator::CreateCommonInvokeLocationSummary(
Nicolas Geoffray4e40c262015-06-03 12:02:38 +0100493 HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100494 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator();
Serban Constantinescu54ff4822016-07-07 18:03:19 +0100495 LocationSummary* locations = new (allocator) LocationSummary(invoke,
496 LocationSummary::kCallOnMainOnly);
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +0100497
498 for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
499 HInstruction* input = invoke->InputAt(i);
500 locations->SetInAt(i, visitor->GetNextLocation(input->GetType()));
501 }
502
503 locations->SetOut(visitor->GetReturnLocation(invoke->GetType()));
Nicolas Geoffray94015b92015-06-04 18:21:04 +0100504
505 if (invoke->IsInvokeStaticOrDirect()) {
506 HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
Vladimir Marko86c87522020-05-11 16:55:55 +0100507 HInvokeStaticOrDirect::MethodLoadKind method_load_kind = call->GetMethodLoadKind();
508 HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = call->GetCodePtrLocation();
509 if (code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallCriticalNative) {
510 locations->AddTemp(Location::RequiresRegister()); // For target method.
511 }
512 if (code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallCriticalNative ||
513 method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kRecursive) {
514 // For `kCallCriticalNative` we need the current method as the hidden argument
515 // if we reach the dlsym lookup stub for @CriticalNative.
516 locations->SetInAt(call->GetCurrentMethodIndex(), visitor->GetMethodLocation());
517 } else {
518 locations->AddTemp(visitor->GetMethodLocation());
519 if (method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall) {
520 locations->SetInAt(call->GetCurrentMethodIndex(), Location::RequiresRegister());
521 }
Nicolas Geoffray94015b92015-06-04 18:21:04 +0100522 }
Orion Hodsoncd260eb2018-06-06 09:04:17 +0100523 } else if (!invoke->IsInvokePolymorphic()) {
Nicolas Geoffray94015b92015-06-04 18:21:04 +0100524 locations->AddTemp(visitor->GetMethodLocation());
525 }
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +0100526}
527
Vladimir Marko86c87522020-05-11 16:55:55 +0100528void CodeGenerator::PrepareCriticalNativeArgumentMoves(
529 HInvokeStaticOrDirect* invoke,
530 /*inout*/InvokeDexCallingConventionVisitor* visitor,
531 /*out*/HParallelMove* parallel_move) {
532 LocationSummary* locations = invoke->GetLocations();
533 for (size_t i = 0, num = invoke->GetNumberOfArguments(); i != num; ++i) {
534 Location in_location = locations->InAt(i);
535 DataType::Type type = invoke->InputAt(i)->GetType();
536 DCHECK_NE(type, DataType::Type::kReference);
537 Location out_location = visitor->GetNextLocation(type);
538 if (out_location.IsStackSlot() || out_location.IsDoubleStackSlot()) {
539 // Stack arguments will need to be moved after adjusting the SP.
540 parallel_move->AddMove(in_location, out_location, type, /*instruction=*/ nullptr);
541 } else {
542 // Register arguments should have been assigned their final locations for register allocation.
543 DCHECK(out_location.Equals(in_location)) << in_location << " -> " << out_location;
544 }
545 }
546}
547
Vladimir Markodec78172020-06-19 15:31:23 +0100548void CodeGenerator::FinishCriticalNativeFrameSetup(size_t out_frame_size,
549 /*inout*/HParallelMove* parallel_move) {
550 DCHECK_NE(out_frame_size, 0u);
551 IncreaseFrame(out_frame_size);
Vladimir Marko86c87522020-05-11 16:55:55 +0100552 // Adjust the source stack offsets by `out_frame_size`, i.e. the additional
553 // frame size needed for outgoing stack arguments.
554 for (size_t i = 0, num = parallel_move->NumMoves(); i != num; ++i) {
555 MoveOperands* operands = parallel_move->MoveOperandsAt(i);
556 Location source = operands->GetSource();
557 if (operands->GetSource().IsStackSlot()) {
558 operands->SetSource(Location::StackSlot(source.GetStackIndex() + out_frame_size));
559 } else if (operands->GetSource().IsDoubleStackSlot()) {
560 operands->SetSource(Location::DoubleStackSlot(source.GetStackIndex() + out_frame_size));
561 }
562 }
Vladimir Markodec78172020-06-19 15:31:23 +0100563 // Emit the moves.
564 GetMoveResolver()->EmitNativeCode(parallel_move);
Vladimir Marko86c87522020-05-11 16:55:55 +0100565}
566
567const char* CodeGenerator::GetCriticalNativeShorty(HInvokeStaticOrDirect* invoke,
568 uint32_t* shorty_len) {
569 ScopedObjectAccess soa(Thread::Current());
570 DCHECK(invoke->GetResolvedMethod()->IsCriticalNative());
571 return invoke->GetResolvedMethod()->GetShorty(shorty_len);
572}
573
Vladimir Markoe7197bf2017-06-02 17:00:23 +0100574void CodeGenerator::GenerateInvokeStaticOrDirectRuntimeCall(
575 HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) {
576 MoveConstant(temp, invoke->GetDexMethodIndex());
577
578 // The access check is unnecessary but we do not want to introduce
579 // extra entrypoints for the codegens that do not support some
580 // invoke type and fall back to the runtime call.
581
582 // Initialize to anything to silent compiler warnings.
583 QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
584 switch (invoke->GetInvokeType()) {
585 case kStatic:
586 entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
587 break;
588 case kDirect:
589 entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
590 break;
591 case kSuper:
592 entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
593 break;
594 case kVirtual:
595 case kInterface:
Orion Hodson43f0cdb2017-10-10 14:47:32 +0100596 case kPolymorphic:
Orion Hodson4c8e12e2018-05-18 08:33:20 +0100597 case kCustom:
Vladimir Markoe7197bf2017-06-02 17:00:23 +0100598 LOG(FATAL) << "Unexpected invoke type: " << invoke->GetInvokeType();
599 UNREACHABLE();
600 }
601
602 InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), slow_path);
603}
Calin Juravle175dc732015-08-25 15:42:32 +0100604void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) {
605 MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex());
606
607 // Initialize to anything to silent compiler warnings.
608 QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
Nicolas Geoffray5e4e11e2016-09-22 13:17:41 +0100609 switch (invoke->GetInvokeType()) {
Calin Juravle175dc732015-08-25 15:42:32 +0100610 case kStatic:
611 entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
612 break;
613 case kDirect:
614 entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
615 break;
616 case kVirtual:
617 entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck;
618 break;
619 case kSuper:
620 entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
621 break;
622 case kInterface:
623 entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck;
624 break;
Orion Hodson43f0cdb2017-10-10 14:47:32 +0100625 case kPolymorphic:
Orion Hodson4c8e12e2018-05-18 08:33:20 +0100626 case kCustom:
Orion Hodson43f0cdb2017-10-10 14:47:32 +0100627 LOG(FATAL) << "Unexpected invoke type: " << invoke->GetInvokeType();
628 UNREACHABLE();
Calin Juravle175dc732015-08-25 15:42:32 +0100629 }
630 InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
631}
632
Orion Hodsonac141392017-01-13 11:53:47 +0000633void CodeGenerator::GenerateInvokePolymorphicCall(HInvokePolymorphic* invoke) {
Orion Hodsoncd260eb2018-06-06 09:04:17 +0100634 // invoke-polymorphic does not use a temporary to convey any additional information (e.g. a
635 // method index) since it requires multiple info from the instruction (registers A, B, H). Not
636 // using the reservation has no effect on the registers used in the runtime call.
Orion Hodsonac141392017-01-13 11:53:47 +0000637 QuickEntrypointEnum entrypoint = kQuickInvokePolymorphic;
638 InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
639}
640
Orion Hodson4c8e12e2018-05-18 08:33:20 +0100641void CodeGenerator::GenerateInvokeCustomCall(HInvokeCustom* invoke) {
642 MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetCallSiteIndex());
643 QuickEntrypointEnum entrypoint = kQuickInvokeCustom;
644 InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
645}
646
Vladimir Marko552a1342017-10-31 10:56:47 +0000647void CodeGenerator::CreateStringBuilderAppendLocations(HStringBuilderAppend* instruction,
648 Location out) {
649 ArenaAllocator* allocator = GetGraph()->GetAllocator();
650 LocationSummary* locations =
651 new (allocator) LocationSummary(instruction, LocationSummary::kCallOnMainOnly);
652 locations->SetOut(out);
653 instruction->GetLocations()->SetInAt(instruction->FormatIndex(),
654 Location::ConstantLocation(instruction->GetFormat()));
655
656 uint32_t format = static_cast<uint32_t>(instruction->GetFormat()->GetValue());
657 uint32_t f = format;
658 PointerSize pointer_size = InstructionSetPointerSize(GetInstructionSet());
659 size_t stack_offset = static_cast<size_t>(pointer_size); // Start after the ArtMethod*.
660 for (size_t i = 0, num_args = instruction->GetNumberOfArguments(); i != num_args; ++i) {
661 StringBuilderAppend::Argument arg_type =
662 static_cast<StringBuilderAppend::Argument>(f & StringBuilderAppend::kArgMask);
663 switch (arg_type) {
664 case StringBuilderAppend::Argument::kStringBuilder:
665 case StringBuilderAppend::Argument::kString:
666 case StringBuilderAppend::Argument::kCharArray:
667 static_assert(sizeof(StackReference<mirror::Object>) == sizeof(uint32_t), "Size check.");
668 FALLTHROUGH_INTENDED;
669 case StringBuilderAppend::Argument::kBoolean:
670 case StringBuilderAppend::Argument::kChar:
671 case StringBuilderAppend::Argument::kInt:
672 case StringBuilderAppend::Argument::kFloat:
673 locations->SetInAt(i, Location::StackSlot(stack_offset));
674 break;
675 case StringBuilderAppend::Argument::kLong:
676 case StringBuilderAppend::Argument::kDouble:
677 stack_offset = RoundUp(stack_offset, sizeof(uint64_t));
678 locations->SetInAt(i, Location::DoubleStackSlot(stack_offset));
679 // Skip the low word, let the common code skip the high word.
680 stack_offset += sizeof(uint32_t);
681 break;
682 default:
683 LOG(FATAL) << "Unexpected arg format: 0x" << std::hex
684 << (f & StringBuilderAppend::kArgMask) << " full format: 0x" << format;
685 UNREACHABLE();
686 }
687 f >>= StringBuilderAppend::kBitsPerArg;
688 stack_offset += sizeof(uint32_t);
689 }
690 DCHECK_EQ(f, 0u);
691
692 size_t param_size = stack_offset - static_cast<size_t>(pointer_size);
693 DCHECK_ALIGNED(param_size, kVRegSize);
694 size_t num_vregs = param_size / kVRegSize;
695 graph_->UpdateMaximumNumberOfOutVRegs(num_vregs);
696}
697
Calin Juravlee460d1d2015-09-29 04:52:17 +0100698void CodeGenerator::CreateUnresolvedFieldLocationSummary(
699 HInstruction* field_access,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100700 DataType::Type field_type,
Calin Juravlee460d1d2015-09-29 04:52:17 +0100701 const FieldAccessCallingConvention& calling_convention) {
702 bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
703 || field_access->IsUnresolvedInstanceFieldSet();
704 bool is_get = field_access->IsUnresolvedInstanceFieldGet()
705 || field_access->IsUnresolvedStaticFieldGet();
706
Vladimir Markoca6fff82017-10-03 14:49:14 +0100707 ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetAllocator();
Calin Juravlee460d1d2015-09-29 04:52:17 +0100708 LocationSummary* locations =
Serban Constantinescu54ff4822016-07-07 18:03:19 +0100709 new (allocator) LocationSummary(field_access, LocationSummary::kCallOnMainOnly);
Calin Juravlee460d1d2015-09-29 04:52:17 +0100710
711 locations->AddTemp(calling_convention.GetFieldIndexLocation());
712
713 if (is_instance) {
714 // Add the `this` object for instance field accesses.
715 locations->SetInAt(0, calling_convention.GetObjectLocation());
716 }
717
718 // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64
719 // regardless of the the type. Because of that we forced to special case
720 // the access to floating point values.
721 if (is_get) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100722 if (DataType::IsFloatingPointType(field_type)) {
Calin Juravlee460d1d2015-09-29 04:52:17 +0100723 // The return value will be stored in regular registers while register
724 // allocator expects it in a floating point register.
725 // Note We don't need to request additional temps because the return
726 // register(s) are already blocked due the call and they may overlap with
727 // the input or field index.
728 // The transfer between the two will be done at codegen level.
729 locations->SetOut(calling_convention.GetFpuLocation(field_type));
730 } else {
731 locations->SetOut(calling_convention.GetReturnLocation(field_type));
732 }
733 } else {
734 size_t set_index = is_instance ? 1 : 0;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100735 if (DataType::IsFloatingPointType(field_type)) {
Calin Juravlee460d1d2015-09-29 04:52:17 +0100736 // The set value comes from a float location while the calling convention
737 // expects it in a regular register location. Allocate a temp for it and
738 // make the transfer at codegen.
739 AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations);
740 locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type));
741 } else {
742 locations->SetInAt(set_index,
743 calling_convention.GetSetValueLocation(field_type, is_instance));
744 }
745 }
746}
747
748void CodeGenerator::GenerateUnresolvedFieldAccess(
749 HInstruction* field_access,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100750 DataType::Type field_type,
Calin Juravlee460d1d2015-09-29 04:52:17 +0100751 uint32_t field_index,
752 uint32_t dex_pc,
753 const FieldAccessCallingConvention& calling_convention) {
754 LocationSummary* locations = field_access->GetLocations();
755
756 MoveConstant(locations->GetTemp(0), field_index);
757
758 bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
759 || field_access->IsUnresolvedInstanceFieldSet();
760 bool is_get = field_access->IsUnresolvedInstanceFieldGet()
761 || field_access->IsUnresolvedStaticFieldGet();
762
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100763 if (!is_get && DataType::IsFloatingPointType(field_type)) {
Calin Juravlee460d1d2015-09-29 04:52:17 +0100764 // Copy the float value to be set into the calling convention register.
765 // Note that using directly the temp location is problematic as we don't
766 // support temp register pairs. To avoid boilerplate conversion code, use
767 // the location from the calling convention.
768 MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance),
769 locations->InAt(is_instance ? 1 : 0),
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100770 (DataType::Is64BitType(field_type) ? DataType::Type::kInt64
771 : DataType::Type::kInt32));
Calin Juravlee460d1d2015-09-29 04:52:17 +0100772 }
773
774 QuickEntrypointEnum entrypoint = kQuickSet8Static; // Initialize to anything to avoid warnings.
775 switch (field_type) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100776 case DataType::Type::kBool:
Calin Juravlee460d1d2015-09-29 04:52:17 +0100777 entrypoint = is_instance
778 ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance)
779 : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static);
780 break;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100781 case DataType::Type::kInt8:
Calin Juravlee460d1d2015-09-29 04:52:17 +0100782 entrypoint = is_instance
783 ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance)
784 : (is_get ? kQuickGetByteStatic : kQuickSet8Static);
785 break;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100786 case DataType::Type::kInt16:
Calin Juravlee460d1d2015-09-29 04:52:17 +0100787 entrypoint = is_instance
788 ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance)
789 : (is_get ? kQuickGetShortStatic : kQuickSet16Static);
790 break;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100791 case DataType::Type::kUint16:
Calin Juravlee460d1d2015-09-29 04:52:17 +0100792 entrypoint = is_instance
793 ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance)
794 : (is_get ? kQuickGetCharStatic : kQuickSet16Static);
795 break;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100796 case DataType::Type::kInt32:
797 case DataType::Type::kFloat32:
Calin Juravlee460d1d2015-09-29 04:52:17 +0100798 entrypoint = is_instance
799 ? (is_get ? kQuickGet32Instance : kQuickSet32Instance)
800 : (is_get ? kQuickGet32Static : kQuickSet32Static);
801 break;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100802 case DataType::Type::kReference:
Calin Juravlee460d1d2015-09-29 04:52:17 +0100803 entrypoint = is_instance
804 ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance)
805 : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic);
806 break;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100807 case DataType::Type::kInt64:
808 case DataType::Type::kFloat64:
Calin Juravlee460d1d2015-09-29 04:52:17 +0100809 entrypoint = is_instance
810 ? (is_get ? kQuickGet64Instance : kQuickSet64Instance)
811 : (is_get ? kQuickGet64Static : kQuickSet64Static);
812 break;
813 default:
814 LOG(FATAL) << "Invalid type " << field_type;
815 }
816 InvokeRuntime(entrypoint, field_access, dex_pc, nullptr);
817
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100818 if (is_get && DataType::IsFloatingPointType(field_type)) {
Calin Juravlee460d1d2015-09-29 04:52:17 +0100819 MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type);
820 }
821}
822
Vladimir Marko41559982017-01-06 14:04:23 +0000823void CodeGenerator::CreateLoadClassRuntimeCallLocationSummary(HLoadClass* cls,
824 Location runtime_type_index_location,
825 Location runtime_return_location) {
Vladimir Marko847e6ce2017-06-02 13:55:07 +0100826 DCHECK_EQ(cls->GetLoadKind(), HLoadClass::LoadKind::kRuntimeCall);
Vladimir Marko41559982017-01-06 14:04:23 +0000827 DCHECK_EQ(cls->InputCount(), 1u);
Vladimir Markoca6fff82017-10-03 14:49:14 +0100828 LocationSummary* locations = new (cls->GetBlock()->GetGraph()->GetAllocator()) LocationSummary(
Vladimir Marko41559982017-01-06 14:04:23 +0000829 cls, LocationSummary::kCallOnMainOnly);
830 locations->SetInAt(0, Location::NoLocation());
831 locations->AddTemp(runtime_type_index_location);
832 locations->SetOut(runtime_return_location);
Calin Juravle98893e12015-10-02 21:05:03 +0100833}
834
Vladimir Marko41559982017-01-06 14:04:23 +0000835void CodeGenerator::GenerateLoadClassRuntimeCall(HLoadClass* cls) {
Vladimir Marko847e6ce2017-06-02 13:55:07 +0100836 DCHECK_EQ(cls->GetLoadKind(), HLoadClass::LoadKind::kRuntimeCall);
Vladimir Markoa9f303c2018-07-20 16:43:56 +0100837 DCHECK(!cls->MustGenerateClinitCheck());
Vladimir Marko41559982017-01-06 14:04:23 +0000838 LocationSummary* locations = cls->GetLocations();
839 MoveConstant(locations->GetTemp(0), cls->GetTypeIndex().index_);
840 if (cls->NeedsAccessCheck()) {
Vladimir Marko9d479252018-07-24 11:35:20 +0100841 CheckEntrypointTypes<kQuickResolveTypeAndVerifyAccess, void*, uint32_t>();
842 InvokeRuntime(kQuickResolveTypeAndVerifyAccess, cls, cls->GetDexPc());
Vladimir Marko41559982017-01-06 14:04:23 +0000843 } else {
Vladimir Marko9d479252018-07-24 11:35:20 +0100844 CheckEntrypointTypes<kQuickResolveType, void*, uint32_t>();
845 InvokeRuntime(kQuickResolveType, cls, cls->GetDexPc());
Vladimir Marko41559982017-01-06 14:04:23 +0000846 }
847}
Calin Juravle98893e12015-10-02 21:05:03 +0100848
Orion Hodsondbaa5c72018-05-10 08:22:46 +0100849void CodeGenerator::CreateLoadMethodHandleRuntimeCallLocationSummary(
850 HLoadMethodHandle* method_handle,
851 Location runtime_proto_index_location,
852 Location runtime_return_location) {
853 DCHECK_EQ(method_handle->InputCount(), 1u);
854 LocationSummary* locations =
855 new (method_handle->GetBlock()->GetGraph()->GetAllocator()) LocationSummary(
856 method_handle, LocationSummary::kCallOnMainOnly);
857 locations->SetInAt(0, Location::NoLocation());
858 locations->AddTemp(runtime_proto_index_location);
859 locations->SetOut(runtime_return_location);
860}
861
862void CodeGenerator::GenerateLoadMethodHandleRuntimeCall(HLoadMethodHandle* method_handle) {
863 LocationSummary* locations = method_handle->GetLocations();
864 MoveConstant(locations->GetTemp(0), method_handle->GetMethodHandleIndex());
865 CheckEntrypointTypes<kQuickResolveMethodHandle, void*, uint32_t>();
866 InvokeRuntime(kQuickResolveMethodHandle, method_handle, method_handle->GetDexPc());
867}
868
Orion Hodson18259d72018-04-12 11:18:23 +0100869void CodeGenerator::CreateLoadMethodTypeRuntimeCallLocationSummary(
870 HLoadMethodType* method_type,
871 Location runtime_proto_index_location,
872 Location runtime_return_location) {
873 DCHECK_EQ(method_type->InputCount(), 1u);
874 LocationSummary* locations =
875 new (method_type->GetBlock()->GetGraph()->GetAllocator()) LocationSummary(
876 method_type, LocationSummary::kCallOnMainOnly);
877 locations->SetInAt(0, Location::NoLocation());
878 locations->AddTemp(runtime_proto_index_location);
879 locations->SetOut(runtime_return_location);
880}
881
882void CodeGenerator::GenerateLoadMethodTypeRuntimeCall(HLoadMethodType* method_type) {
883 LocationSummary* locations = method_type->GetLocations();
Orion Hodson06d10a72018-05-14 08:53:38 +0100884 MoveConstant(locations->GetTemp(0), method_type->GetProtoIndex().index_);
Orion Hodson18259d72018-04-12 11:18:23 +0100885 CheckEntrypointTypes<kQuickResolveMethodType, void*, uint32_t>();
886 InvokeRuntime(kQuickResolveMethodType, method_type, method_type->GetDexPc());
887}
888
Vladimir Markoe47f60c2018-02-21 13:43:28 +0000889static uint32_t GetBootImageOffsetImpl(const void* object, ImageHeader::ImageSections section) {
890 Runtime* runtime = Runtime::Current();
891 DCHECK(runtime->IsAotCompiler());
892 const std::vector<gc::space::ImageSpace*>& boot_image_spaces =
893 runtime->GetHeap()->GetBootImageSpaces();
894 // Check that the `object` is in the expected section of one of the boot image files.
895 DCHECK(std::any_of(boot_image_spaces.begin(),
896 boot_image_spaces.end(),
897 [object, section](gc::space::ImageSpace* space) {
898 uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin());
899 uintptr_t offset = reinterpret_cast<uintptr_t>(object) - begin;
900 return space->GetImageHeader().GetImageSection(section).Contains(offset);
901 }));
902 uintptr_t begin = reinterpret_cast<uintptr_t>(boot_image_spaces.front()->Begin());
903 uintptr_t offset = reinterpret_cast<uintptr_t>(object) - begin;
904 return dchecked_integral_cast<uint32_t>(offset);
905}
906
907// NO_THREAD_SAFETY_ANALYSIS: Avoid taking the mutator lock, boot image classes are non-moveable.
908uint32_t CodeGenerator::GetBootImageOffset(HLoadClass* load_class) NO_THREAD_SAFETY_ANALYSIS {
909 DCHECK_EQ(load_class->GetLoadKind(), HLoadClass::LoadKind::kBootImageRelRo);
910 ObjPtr<mirror::Class> klass = load_class->GetClass().Get();
911 DCHECK(klass != nullptr);
912 return GetBootImageOffsetImpl(klass.Ptr(), ImageHeader::kSectionObjects);
913}
914
915// NO_THREAD_SAFETY_ANALYSIS: Avoid taking the mutator lock, boot image strings are non-moveable.
916uint32_t CodeGenerator::GetBootImageOffset(HLoadString* load_string) NO_THREAD_SAFETY_ANALYSIS {
917 DCHECK_EQ(load_string->GetLoadKind(), HLoadString::LoadKind::kBootImageRelRo);
918 ObjPtr<mirror::String> string = load_string->GetString().Get();
919 DCHECK(string != nullptr);
920 return GetBootImageOffsetImpl(string.Ptr(), ImageHeader::kSectionObjects);
921}
922
923uint32_t CodeGenerator::GetBootImageOffset(HInvokeStaticOrDirect* invoke) {
924 DCHECK_EQ(invoke->GetMethodLoadKind(), HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo);
925 ArtMethod* method = invoke->GetResolvedMethod();
926 DCHECK(method != nullptr);
927 return GetBootImageOffsetImpl(method, ImageHeader::kSectionArtMethods);
928}
929
Mark Mendell5f874182015-03-04 15:42:45 -0500930void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const {
931 // The DCHECKS below check that a register is not specified twice in
932 // the summary. The out location can overlap with an input, so we need
933 // to special case it.
934 if (location.IsRegister()) {
935 DCHECK(is_out || !blocked_core_registers_[location.reg()]);
936 blocked_core_registers_[location.reg()] = true;
937 } else if (location.IsFpuRegister()) {
938 DCHECK(is_out || !blocked_fpu_registers_[location.reg()]);
939 blocked_fpu_registers_[location.reg()] = true;
940 } else if (location.IsFpuRegisterPair()) {
941 DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]);
942 blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true;
943 DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]);
944 blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true;
945 } else if (location.IsRegisterPair()) {
946 DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]);
947 blocked_core_registers_[location.AsRegisterPairLow<int>()] = true;
948 DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]);
949 blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true;
950 }
951}
952
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000953void CodeGenerator::AllocateLocations(HInstruction* instruction) {
Vladimir Markoec32f642017-06-02 10:51:55 +0100954 for (HEnvironment* env = instruction->GetEnvironment(); env != nullptr; env = env->GetParent()) {
955 env->AllocateLocations();
956 }
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000957 instruction->Accept(GetLocationBuilder());
Alexandre Rames88c13cd2015-04-14 17:35:39 +0100958 DCHECK(CheckTypeConsistency(instruction));
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000959 LocationSummary* locations = instruction->GetLocations();
960 if (!instruction->IsSuspendCheckEntry()) {
Aart Bikd1c40452016-03-02 16:06:13 -0800961 if (locations != nullptr) {
962 if (locations->CanCall()) {
963 MarkNotLeaf();
964 } else if (locations->Intrinsified() &&
965 instruction->IsInvokeStaticOrDirect() &&
966 !instruction->AsInvokeStaticOrDirect()->HasCurrentMethodInput()) {
967 // A static method call that has been fully intrinsified, and cannot call on the slow
968 // path or refer to the current method directly, no longer needs current method.
969 return;
970 }
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000971 }
972 if (instruction->NeedsCurrentMethod()) {
973 SetRequiresCurrentMethod();
974 }
975 }
976}
977
Vladimir Markod58b8372016-04-12 18:51:43 +0100978std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph,
Vladimir Markod58b8372016-04-12 18:51:43 +0100979 const CompilerOptions& compiler_options,
980 OptimizingCompilerStats* stats) {
Vladimir Markoe764d2e2017-10-05 14:35:55 +0100981 ArenaAllocator* allocator = graph->GetAllocator();
Vladimir Markoa0431112018-06-25 09:32:54 +0100982 switch (compiler_options.GetInstructionSet()) {
Alex Light50fa9932015-08-10 15:30:07 -0700983#ifdef ART_ENABLE_CODEGEN_arm
Vladimir Marko33bff252017-11-01 14:35:42 +0000984 case InstructionSet::kArm:
985 case InstructionSet::kThumb2: {
Roland Levillain9983e302017-07-14 14:34:22 +0100986 return std::unique_ptr<CodeGenerator>(
Vladimir Markoa0431112018-06-25 09:32:54 +0100987 new (allocator) arm::CodeGeneratorARMVIXL(graph, compiler_options, stats));
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +0000988 }
Alex Light50fa9932015-08-10 15:30:07 -0700989#endif
990#ifdef ART_ENABLE_CODEGEN_arm64
Vladimir Marko33bff252017-11-01 14:35:42 +0000991 case InstructionSet::kArm64: {
Vladimir Markod58b8372016-04-12 18:51:43 +0100992 return std::unique_ptr<CodeGenerator>(
Vladimir Markoa0431112018-06-25 09:32:54 +0100993 new (allocator) arm64::CodeGeneratorARM64(graph, compiler_options, stats));
Alexandre Rames5319def2014-10-23 10:03:10 +0100994 }
Alex Light50fa9932015-08-10 15:30:07 -0700995#endif
Alex Light50fa9932015-08-10 15:30:07 -0700996#ifdef ART_ENABLE_CODEGEN_x86
Vladimir Marko33bff252017-11-01 14:35:42 +0000997 case InstructionSet::kX86: {
Vladimir Markod58b8372016-04-12 18:51:43 +0100998 return std::unique_ptr<CodeGenerator>(
Vladimir Markoa0431112018-06-25 09:32:54 +0100999 new (allocator) x86::CodeGeneratorX86(graph, compiler_options, stats));
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +00001000 }
Alex Light50fa9932015-08-10 15:30:07 -07001001#endif
1002#ifdef ART_ENABLE_CODEGEN_x86_64
Vladimir Marko33bff252017-11-01 14:35:42 +00001003 case InstructionSet::kX86_64: {
Vladimir Markod58b8372016-04-12 18:51:43 +01001004 return std::unique_ptr<CodeGenerator>(
Vladimir Markoa0431112018-06-25 09:32:54 +01001005 new (allocator) x86_64::CodeGeneratorX86_64(graph, compiler_options, stats));
Dmitry Petrochenko6a58cb12014-04-02 17:27:59 +07001006 }
Alex Light50fa9932015-08-10 15:30:07 -07001007#endif
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +00001008 default:
Nicolas Geoffray787c3072014-03-17 10:20:19 +00001009 return nullptr;
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +00001010 }
1011}
1012
Vladimir Marko174b2e22017-10-12 13:34:49 +01001013CodeGenerator::CodeGenerator(HGraph* graph,
1014 size_t number_of_core_registers,
1015 size_t number_of_fpu_registers,
1016 size_t number_of_register_pairs,
1017 uint32_t core_callee_save_mask,
1018 uint32_t fpu_callee_save_mask,
1019 const CompilerOptions& compiler_options,
1020 OptimizingCompilerStats* stats)
1021 : frame_size_(0),
1022 core_spill_mask_(0),
1023 fpu_spill_mask_(0),
1024 first_register_slot_in_slow_path_(0),
1025 allocated_registers_(RegisterSet::Empty()),
1026 blocked_core_registers_(graph->GetAllocator()->AllocArray<bool>(number_of_core_registers,
1027 kArenaAllocCodeGenerator)),
1028 blocked_fpu_registers_(graph->GetAllocator()->AllocArray<bool>(number_of_fpu_registers,
1029 kArenaAllocCodeGenerator)),
1030 number_of_core_registers_(number_of_core_registers),
1031 number_of_fpu_registers_(number_of_fpu_registers),
1032 number_of_register_pairs_(number_of_register_pairs),
1033 core_callee_save_mask_(core_callee_save_mask),
1034 fpu_callee_save_mask_(fpu_callee_save_mask),
1035 block_order_(nullptr),
1036 disasm_info_(nullptr),
1037 stats_(stats),
1038 graph_(graph),
1039 compiler_options_(compiler_options),
1040 current_slow_path_(nullptr),
1041 current_block_index_(0),
1042 is_leaf_(true),
1043 requires_current_method_(false),
1044 code_generation_data_() {
Nicolas Geoffray57cacb72019-12-08 22:07:08 +00001045 if (GetGraph()->IsCompilingOsr()) {
1046 // Make OSR methods have all registers spilled, this simplifies the logic of
1047 // jumping to the compiled code directly.
1048 for (size_t i = 0; i < number_of_core_registers_; ++i) {
1049 if (IsCoreCalleeSaveRegister(i)) {
1050 AddAllocatedRegister(Location::RegisterLocation(i));
1051 }
1052 }
1053 for (size_t i = 0; i < number_of_fpu_registers_; ++i) {
1054 if (IsFloatingPointCalleeSaveRegister(i)) {
1055 AddAllocatedRegister(Location::FpuRegisterLocation(i));
1056 }
1057 }
1058 }
Vladimir Marko174b2e22017-10-12 13:34:49 +01001059}
1060
1061CodeGenerator::~CodeGenerator() {}
1062
Vladimir Marko174b2e22017-10-12 13:34:49 +01001063size_t CodeGenerator::GetNumberOfJitRoots() const {
1064 DCHECK(code_generation_data_ != nullptr);
1065 return code_generation_data_->GetNumberOfJitRoots();
Nicolas Geoffrayd28b9692015-11-04 14:36:55 +00001066}
1067
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001068static void CheckCovers(uint32_t dex_pc,
1069 const HGraph& graph,
1070 const CodeInfo& code_info,
1071 const ArenaVector<HSuspendCheck*>& loop_headers,
1072 ArenaVector<size_t>* covered) {
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001073 for (size_t i = 0; i < loop_headers.size(); ++i) {
1074 if (loop_headers[i]->GetDexPc() == dex_pc) {
1075 if (graph.IsCompilingOsr()) {
David Srbecky052f8ca2018-04-26 15:42:54 +01001076 DCHECK(code_info.GetOsrStackMapForDexPc(dex_pc).IsValid());
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001077 }
1078 ++(*covered)[i];
1079 }
1080 }
1081}
1082
1083// Debug helper to ensure loop entries in compiled code are matched by
1084// dex branch instructions.
1085static void CheckLoopEntriesCanBeUsedForOsr(const HGraph& graph,
1086 const CodeInfo& code_info,
Andreas Gampe3f1dcd32018-12-28 09:39:56 -08001087 const dex::CodeItem& code_item) {
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001088 if (graph.HasTryCatch()) {
1089 // One can write loops through try/catch, which we do not support for OSR anyway.
1090 return;
1091 }
Vladimir Markoca6fff82017-10-03 14:49:14 +01001092 ArenaVector<HSuspendCheck*> loop_headers(graph.GetAllocator()->Adapter(kArenaAllocMisc));
Vladimir Marko2c45bc92016-10-25 16:54:12 +01001093 for (HBasicBlock* block : graph.GetReversePostOrder()) {
1094 if (block->IsLoopHeader()) {
1095 HSuspendCheck* suspend_check = block->GetLoopInformation()->GetSuspendCheck();
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001096 if (!suspend_check->GetEnvironment()->IsFromInlinedInvoke()) {
1097 loop_headers.push_back(suspend_check);
1098 }
1099 }
1100 }
Vladimir Markoca6fff82017-10-03 14:49:14 +01001101 ArenaVector<size_t> covered(
1102 loop_headers.size(), 0, graph.GetAllocator()->Adapter(kArenaAllocMisc));
Mathieu Chartier698ebbc2018-01-05 11:00:42 -08001103 for (const DexInstructionPcPair& pair : CodeItemInstructionAccessor(graph.GetDexFile(),
Mathieu Chartier73f21d42018-01-02 14:26:50 -08001104 &code_item)) {
Mathieu Chartier2b2bef22017-10-26 17:10:19 -07001105 const uint32_t dex_pc = pair.DexPc();
1106 const Instruction& instruction = pair.Inst();
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001107 if (instruction.IsBranch()) {
1108 uint32_t target = dex_pc + instruction.GetTargetOffset();
1109 CheckCovers(target, graph, code_info, loop_headers, &covered);
1110 } else if (instruction.IsSwitch()) {
David Brazdil86ea7ee2016-02-16 09:26:07 +00001111 DexSwitchTable table(instruction, dex_pc);
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001112 uint16_t num_entries = table.GetNumEntries();
1113 size_t offset = table.GetFirstValueIndex();
1114
1115 // Use a larger loop counter type to avoid overflow issues.
1116 for (size_t i = 0; i < num_entries; ++i) {
1117 // The target of the case.
1118 uint32_t target = dex_pc + table.GetEntryAt(i + offset);
1119 CheckCovers(target, graph, code_info, loop_headers, &covered);
1120 }
1121 }
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001122 }
1123
1124 for (size_t i = 0; i < covered.size(); ++i) {
1125 DCHECK_NE(covered[i], 0u) << "Loop in compiled code has no dex branch equivalent";
1126 }
1127}
1128
Andreas Gampe3f1dcd32018-12-28 09:39:56 -08001129ScopedArenaVector<uint8_t> CodeGenerator::BuildStackMaps(const dex::CodeItem* code_item) {
David Srbeckye7a91942018-08-01 17:23:53 +01001130 ScopedArenaVector<uint8_t> stack_map = GetStackMapStream()->Encode();
1131 if (kIsDebugBuild && code_item != nullptr) {
1132 CheckLoopEntriesCanBeUsedForOsr(*graph_, CodeInfo(stack_map.data()), *code_item);
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001133 }
David Srbeckye7a91942018-08-01 17:23:53 +01001134 return stack_map;
Nicolas Geoffray39468442014-09-02 15:17:15 +01001135}
1136
Artem Serov2808be82018-12-20 19:15:11 +00001137// Returns whether stackmap dex register info is needed for the instruction.
1138//
1139// The following cases mandate having a dex register map:
1140// * Deoptimization
1141// when we need to obtain the values to restore actual vregisters for interpreter.
1142// * Debuggability
1143// when we want to observe the values / asynchronously deoptimize.
1144// * Monitor operations
1145// to allow dumping in a stack trace locked dex registers for non-debuggable code.
1146// * On-stack-replacement (OSR)
1147// when entering compiled for OSR code from the interpreter we need to initialize the compiled
1148// code values with the values from the vregisters.
1149// * Method local catch blocks
1150// a catch block must see the environment of the instruction from the same method that can
1151// throw to this block.
1152static bool NeedsVregInfo(HInstruction* instruction, bool osr) {
1153 HGraph* graph = instruction->GetBlock()->GetGraph();
1154 return instruction->IsDeoptimize() ||
1155 graph->IsDebuggable() ||
1156 graph->HasMonitorOperations() ||
1157 osr ||
1158 instruction->CanThrowIntoCatchBlock();
1159}
1160
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001161void CodeGenerator::RecordPcInfo(HInstruction* instruction,
1162 uint32_t dex_pc,
David Srbecky50fac062018-06-13 18:55:35 +01001163 SlowPathCode* slow_path,
1164 bool native_debug_info) {
Evgeny Astigeevich98416bf2019-09-09 14:52:12 +01001165 RecordPcInfo(instruction, dex_pc, GetAssembler()->CodePosition(), slow_path, native_debug_info);
1166}
1167
1168void CodeGenerator::RecordPcInfo(HInstruction* instruction,
1169 uint32_t dex_pc,
1170 uint32_t native_pc,
1171 SlowPathCode* slow_path,
1172 bool native_debug_info) {
Calin Juravled2ec87d2014-12-08 14:24:46 +00001173 if (instruction != nullptr) {
Roland Levillain1693a1f2016-03-15 14:57:31 +00001174 // The code generated for some type conversions
Alexey Frunze4dda3372015-06-01 18:31:49 -07001175 // may call the runtime, thus normally requiring a subsequent
1176 // call to this method. However, the method verifier does not
1177 // produce PC information for certain instructions, which are
1178 // considered "atomic" (they cannot join a GC).
Roland Levillain624279f2014-12-04 11:54:28 +00001179 // Therefore we do not currently record PC information for such
1180 // instructions. As this may change later, we added this special
1181 // case so that code generators may nevertheless call
1182 // CodeGenerator::RecordPcInfo without triggering an error in
1183 // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x")
1184 // thereafter.
Roland Levillain1693a1f2016-03-15 14:57:31 +00001185 if (instruction->IsTypeConversion()) {
Calin Juravled2ec87d2014-12-08 14:24:46 +00001186 return;
1187 }
1188 if (instruction->IsRem()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001189 DataType::Type type = instruction->AsRem()->GetResultType();
1190 if ((type == DataType::Type::kFloat32) || (type == DataType::Type::kFloat64)) {
Calin Juravled2ec87d2014-12-08 14:24:46 +00001191 return;
1192 }
1193 }
Roland Levillain624279f2014-12-04 11:54:28 +00001194 }
1195
Vladimir Marko174b2e22017-10-12 13:34:49 +01001196 StackMapStream* stack_map_stream = GetStackMapStream();
Nicolas Geoffray39468442014-09-02 15:17:15 +01001197 if (instruction == nullptr) {
David Srbeckyc7098ff2016-02-09 14:30:11 +00001198 // For stack overflow checks and native-debug-info entries without dex register
1199 // mapping (i.e. start of basic block or start of slow path).
David Srbeckyf6ba5b32018-06-23 22:05:49 +01001200 stack_map_stream->BeginStackMapEntry(dex_pc, native_pc);
Vladimir Marko174b2e22017-10-12 13:34:49 +01001201 stack_map_stream->EndStackMapEntry();
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001202 return;
1203 }
Nicolas Geoffray39468442014-09-02 15:17:15 +01001204
Vladimir Markofec85cd2017-12-04 13:00:12 +00001205 LocationSummary* locations = instruction->GetLocations();
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001206 uint32_t register_mask = locations->GetRegisterMask();
Vladimir Marko70e97462016-08-09 11:04:26 +01001207 DCHECK_EQ(register_mask & ~locations->GetLiveRegisters()->GetCoreRegisters(), 0u);
1208 if (locations->OnlyCallsOnSlowPath()) {
1209 // In case of slow path, we currently set the location of caller-save registers
1210 // to register (instead of their stack location when pushed before the slow-path
1211 // call). Therefore register_mask contains both callee-save and caller-save
1212 // registers that hold objects. We must remove the spilled caller-save from the
1213 // mask, since they will be overwritten by the callee.
Andreas Gampe3db70682018-12-26 15:12:03 -08001214 uint32_t spills = GetSlowPathSpills(locations, /* core_registers= */ true);
Vladimir Marko70e97462016-08-09 11:04:26 +01001215 register_mask &= ~spills;
Vladimir Marko952dbb12016-07-28 12:01:51 +01001216 } else {
Vladimir Marko952dbb12016-07-28 12:01:51 +01001217 // The register mask must be a subset of callee-save registers.
1218 DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001219 }
Vladimir Markofec85cd2017-12-04 13:00:12 +00001220
1221 uint32_t outer_dex_pc = dex_pc;
1222 uint32_t outer_environment_size = 0u;
1223 uint32_t inlining_depth = 0;
1224 HEnvironment* const environment = instruction->GetEnvironment();
1225 if (environment != nullptr) {
1226 HEnvironment* outer_environment = environment;
1227 while (outer_environment->GetParent() != nullptr) {
1228 outer_environment = outer_environment->GetParent();
1229 ++inlining_depth;
1230 }
1231 outer_dex_pc = outer_environment->GetDexPc();
1232 outer_environment_size = outer_environment->Size();
1233 }
David Srbecky50fac062018-06-13 18:55:35 +01001234
1235 HLoopInformation* info = instruction->GetBlock()->GetLoopInformation();
1236 bool osr =
1237 instruction->IsSuspendCheck() &&
1238 (info != nullptr) &&
1239 graph_->IsCompilingOsr() &&
1240 (inlining_depth == 0);
1241 StackMap::Kind kind = native_debug_info
1242 ? StackMap::Kind::Debug
1243 : (osr ? StackMap::Kind::OSR : StackMap::Kind::Default);
Artem Serov2808be82018-12-20 19:15:11 +00001244 bool needs_vreg_info = NeedsVregInfo(instruction, osr);
Vladimir Marko174b2e22017-10-12 13:34:49 +01001245 stack_map_stream->BeginStackMapEntry(outer_dex_pc,
Vladimir Markobd8c7252015-06-12 10:06:32 +01001246 native_pc,
Calin Juravle4f46ac52015-04-23 18:47:21 +01001247 register_mask,
1248 locations->GetStackMask(),
Artem Serov2808be82018-12-20 19:15:11 +00001249 kind,
1250 needs_vreg_info);
1251
1252 EmitEnvironment(environment, slow_path, needs_vreg_info);
Vladimir Marko174b2e22017-10-12 13:34:49 +01001253 stack_map_stream->EndStackMapEntry();
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001254
David Srbecky50fac062018-06-13 18:55:35 +01001255 if (osr) {
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001256 DCHECK_EQ(info->GetSuspendCheck(), instruction);
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001257 DCHECK(info->IsIrreducible());
Andreas Gampe875b4f22018-11-19 12:59:15 -08001258 DCHECK(environment != nullptr);
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001259 if (kIsDebugBuild) {
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001260 for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
1261 HInstruction* in_environment = environment->GetInstructionAt(i);
1262 if (in_environment != nullptr) {
1263 DCHECK(in_environment->IsPhi() || in_environment->IsConstant());
1264 Location location = environment->GetLocationAt(i);
1265 DCHECK(location.IsStackSlot() ||
1266 location.IsDoubleStackSlot() ||
1267 location.IsConstant() ||
1268 location.IsInvalid());
1269 if (location.IsStackSlot() || location.IsDoubleStackSlot()) {
1270 DCHECK_LT(location.GetStackIndex(), static_cast<int32_t>(GetFrameSize()));
1271 }
1272 }
1273 }
1274 }
Nicolas Geoffrayb331feb2016-02-05 16:51:53 +00001275 }
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001276}
1277
David Srbeckyb7070a22016-01-08 18:13:53 +00001278bool CodeGenerator::HasStackMapAtCurrentPc() {
1279 uint32_t pc = GetAssembler()->CodeSize();
Vladimir Marko174b2e22017-10-12 13:34:49 +01001280 StackMapStream* stack_map_stream = GetStackMapStream();
1281 size_t count = stack_map_stream->GetNumberOfStackMaps();
Andreas Gampe0c95c122017-02-26 14:10:28 -08001282 if (count == 0) {
1283 return false;
1284 }
David Srbeckyd02b23f2018-05-29 23:27:22 +01001285 return stack_map_stream->GetStackMapNativePcOffset(count - 1) == pc;
David Srbeckyb7070a22016-01-08 18:13:53 +00001286}
1287
David Srbeckyd28f4a02016-03-14 17:14:24 +00001288void CodeGenerator::MaybeRecordNativeDebugInfo(HInstruction* instruction,
1289 uint32_t dex_pc,
1290 SlowPathCode* slow_path) {
David Srbeckyc7098ff2016-02-09 14:30:11 +00001291 if (GetCompilerOptions().GetNativeDebuggable() && dex_pc != kNoDexPc) {
1292 if (HasStackMapAtCurrentPc()) {
1293 // Ensure that we do not collide with the stack map of the previous instruction.
1294 GenerateNop();
1295 }
Andreas Gampe3db70682018-12-26 15:12:03 -08001296 RecordPcInfo(instruction, dex_pc, slow_path, /* native_debug_info= */ true);
David Srbeckyc7098ff2016-02-09 14:30:11 +00001297 }
1298}
1299
David Brazdil77a48ae2015-09-15 12:34:04 +00001300void CodeGenerator::RecordCatchBlockInfo() {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001301 StackMapStream* stack_map_stream = GetStackMapStream();
David Brazdil77a48ae2015-09-15 12:34:04 +00001302
Vladimir Markofa6b93c2015-09-15 10:15:55 +01001303 for (HBasicBlock* block : *block_order_) {
David Brazdil77a48ae2015-09-15 12:34:04 +00001304 if (!block->IsCatchBlock()) {
1305 continue;
1306 }
1307
1308 uint32_t dex_pc = block->GetDexPc();
1309 uint32_t num_vregs = graph_->GetNumberOfVRegs();
David Brazdil77a48ae2015-09-15 12:34:04 +00001310 uint32_t native_pc = GetAddressOf(block);
David Brazdil77a48ae2015-09-15 12:34:04 +00001311
Vladimir Marko174b2e22017-10-12 13:34:49 +01001312 stack_map_stream->BeginStackMapEntry(dex_pc,
David Brazdil77a48ae2015-09-15 12:34:04 +00001313 native_pc,
Andreas Gampe3db70682018-12-26 15:12:03 -08001314 /* register_mask= */ 0,
1315 /* sp_mask= */ nullptr,
David Srbecky50fac062018-06-13 18:55:35 +01001316 StackMap::Kind::Catch);
David Brazdil77a48ae2015-09-15 12:34:04 +00001317
1318 HInstruction* current_phi = block->GetFirstPhi();
1319 for (size_t vreg = 0; vreg < num_vregs; ++vreg) {
David Srbecky50fac062018-06-13 18:55:35 +01001320 while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) {
1321 HInstruction* next_phi = current_phi->GetNext();
1322 DCHECK(next_phi == nullptr ||
1323 current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber())
1324 << "Phis need to be sorted by vreg number to keep this a linear-time loop.";
1325 current_phi = next_phi;
1326 }
David Brazdil77a48ae2015-09-15 12:34:04 +00001327
1328 if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001329 stack_map_stream->AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
David Brazdil77a48ae2015-09-15 12:34:04 +00001330 } else {
Vladimir Markobea75ff2017-10-11 20:39:54 +01001331 Location location = current_phi->GetLocations()->Out();
David Brazdil77a48ae2015-09-15 12:34:04 +00001332 switch (location.GetKind()) {
1333 case Location::kStackSlot: {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001334 stack_map_stream->AddDexRegisterEntry(
David Brazdil77a48ae2015-09-15 12:34:04 +00001335 DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
1336 break;
1337 }
1338 case Location::kDoubleStackSlot: {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001339 stack_map_stream->AddDexRegisterEntry(
David Brazdil77a48ae2015-09-15 12:34:04 +00001340 DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
Vladimir Marko174b2e22017-10-12 13:34:49 +01001341 stack_map_stream->AddDexRegisterEntry(
David Brazdil77a48ae2015-09-15 12:34:04 +00001342 DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
1343 ++vreg;
1344 DCHECK_LT(vreg, num_vregs);
1345 break;
1346 }
1347 default: {
1348 // All catch phis must be allocated to a stack slot.
1349 LOG(FATAL) << "Unexpected kind " << location.GetKind();
1350 UNREACHABLE();
1351 }
1352 }
1353 }
1354 }
1355
Vladimir Marko174b2e22017-10-12 13:34:49 +01001356 stack_map_stream->EndStackMapEntry();
David Brazdil77a48ae2015-09-15 12:34:04 +00001357 }
1358}
1359
Vladimir Marko174b2e22017-10-12 13:34:49 +01001360void CodeGenerator::AddSlowPath(SlowPathCode* slow_path) {
1361 DCHECK(code_generation_data_ != nullptr);
1362 code_generation_data_->AddSlowPath(slow_path);
1363}
1364
Artem Serov2808be82018-12-20 19:15:11 +00001365void CodeGenerator::EmitVRegInfo(HEnvironment* environment, SlowPathCode* slow_path) {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001366 StackMapStream* stack_map_stream = GetStackMapStream();
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001367 // Walk over the environment, and record the location of dex registers.
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001368 for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001369 HInstruction* current = environment->GetInstructionAt(i);
1370 if (current == nullptr) {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001371 stack_map_stream->AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001372 continue;
Nicolas Geoffray39468442014-09-02 15:17:15 +01001373 }
1374
David Srbeckye1402122018-06-13 18:20:45 +01001375 using Kind = DexRegisterLocation::Kind;
Nicolas Geoffray0a23d742015-05-07 11:57:35 +01001376 Location location = environment->GetLocationAt(i);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001377 switch (location.GetKind()) {
1378 case Location::kConstant: {
1379 DCHECK_EQ(current, location.GetConstant());
1380 if (current->IsLongConstant()) {
1381 int64_t value = current->AsLongConstant()->GetValue();
David Srbeckye1402122018-06-13 18:20:45 +01001382 stack_map_stream->AddDexRegisterEntry(Kind::kConstant, Low32Bits(value));
1383 stack_map_stream->AddDexRegisterEntry(Kind::kConstant, High32Bits(value));
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001384 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001385 DCHECK_LT(i, environment_size);
1386 } else if (current->IsDoubleConstant()) {
Roland Levillainda4d79b2015-03-24 14:36:11 +00001387 int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue());
David Srbeckye1402122018-06-13 18:20:45 +01001388 stack_map_stream->AddDexRegisterEntry(Kind::kConstant, Low32Bits(value));
1389 stack_map_stream->AddDexRegisterEntry(Kind::kConstant, High32Bits(value));
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001390 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001391 DCHECK_LT(i, environment_size);
1392 } else if (current->IsIntConstant()) {
1393 int32_t value = current->AsIntConstant()->GetValue();
David Srbeckye1402122018-06-13 18:20:45 +01001394 stack_map_stream->AddDexRegisterEntry(Kind::kConstant, value);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001395 } else if (current->IsNullConstant()) {
David Srbeckye1402122018-06-13 18:20:45 +01001396 stack_map_stream->AddDexRegisterEntry(Kind::kConstant, 0);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001397 } else {
1398 DCHECK(current->IsFloatConstant()) << current->DebugName();
Roland Levillainda4d79b2015-03-24 14:36:11 +00001399 int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue());
David Srbeckye1402122018-06-13 18:20:45 +01001400 stack_map_stream->AddDexRegisterEntry(Kind::kConstant, value);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001401 }
1402 break;
1403 }
1404
1405 case Location::kStackSlot: {
David Srbeckye1402122018-06-13 18:20:45 +01001406 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, location.GetStackIndex());
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001407 break;
1408 }
1409
1410 case Location::kDoubleStackSlot: {
David Srbeckye1402122018-06-13 18:20:45 +01001411 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, location.GetStackIndex());
Vladimir Marko174b2e22017-10-12 13:34:49 +01001412 stack_map_stream->AddDexRegisterEntry(
David Srbeckye1402122018-06-13 18:20:45 +01001413 Kind::kInStack, location.GetHighStackIndex(kVRegSize));
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001414 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001415 DCHECK_LT(i, environment_size);
1416 break;
1417 }
1418
1419 case Location::kRegister : {
1420 int id = location.reg();
1421 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) {
1422 uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id);
David Srbeckye1402122018-06-13 18:20:45 +01001423 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001424 if (current->GetType() == DataType::Type::kInt64) {
David Srbeckye1402122018-06-13 18:20:45 +01001425 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset + kVRegSize);
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001426 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001427 DCHECK_LT(i, environment_size);
1428 }
1429 } else {
David Srbeckye1402122018-06-13 18:20:45 +01001430 stack_map_stream->AddDexRegisterEntry(Kind::kInRegister, id);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001431 if (current->GetType() == DataType::Type::kInt64) {
David Srbeckye1402122018-06-13 18:20:45 +01001432 stack_map_stream->AddDexRegisterEntry(Kind::kInRegisterHigh, id);
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001433 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001434 DCHECK_LT(i, environment_size);
1435 }
1436 }
1437 break;
1438 }
1439
1440 case Location::kFpuRegister : {
1441 int id = location.reg();
1442 if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) {
1443 uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id);
David Srbeckye1402122018-06-13 18:20:45 +01001444 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001445 if (current->GetType() == DataType::Type::kFloat64) {
David Srbeckye1402122018-06-13 18:20:45 +01001446 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset + kVRegSize);
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001447 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001448 DCHECK_LT(i, environment_size);
1449 }
1450 } else {
David Srbeckye1402122018-06-13 18:20:45 +01001451 stack_map_stream->AddDexRegisterEntry(Kind::kInFpuRegister, id);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001452 if (current->GetType() == DataType::Type::kFloat64) {
David Srbeckye1402122018-06-13 18:20:45 +01001453 stack_map_stream->AddDexRegisterEntry(Kind::kInFpuRegisterHigh, id);
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001454 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001455 DCHECK_LT(i, environment_size);
1456 }
1457 }
1458 break;
1459 }
1460
1461 case Location::kFpuRegisterPair : {
1462 int low = location.low();
1463 int high = location.high();
1464 if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) {
1465 uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low);
David Srbeckye1402122018-06-13 18:20:45 +01001466 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001467 } else {
David Srbeckye1402122018-06-13 18:20:45 +01001468 stack_map_stream->AddDexRegisterEntry(Kind::kInFpuRegister, low);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001469 }
1470 if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) {
1471 uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high);
David Srbeckye1402122018-06-13 18:20:45 +01001472 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset);
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001473 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001474 } else {
David Srbeckye1402122018-06-13 18:20:45 +01001475 stack_map_stream->AddDexRegisterEntry(Kind::kInFpuRegister, high);
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001476 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001477 }
1478 DCHECK_LT(i, environment_size);
1479 break;
1480 }
1481
1482 case Location::kRegisterPair : {
1483 int low = location.low();
1484 int high = location.high();
1485 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) {
1486 uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low);
David Srbeckye1402122018-06-13 18:20:45 +01001487 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001488 } else {
David Srbeckye1402122018-06-13 18:20:45 +01001489 stack_map_stream->AddDexRegisterEntry(Kind::kInRegister, low);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001490 }
1491 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) {
1492 uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high);
David Srbeckye1402122018-06-13 18:20:45 +01001493 stack_map_stream->AddDexRegisterEntry(Kind::kInStack, offset);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001494 } else {
David Srbeckye1402122018-06-13 18:20:45 +01001495 stack_map_stream->AddDexRegisterEntry(Kind::kInRegister, high);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001496 }
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001497 ++i;
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001498 DCHECK_LT(i, environment_size);
1499 break;
1500 }
1501
1502 case Location::kInvalid: {
David Srbeckye1402122018-06-13 18:20:45 +01001503 stack_map_stream->AddDexRegisterEntry(Kind::kNone, 0);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001504 break;
1505 }
1506
1507 default:
1508 LOG(FATAL) << "Unexpected kind " << location.GetKind();
1509 }
Nicolas Geoffray39468442014-09-02 15:17:15 +01001510 }
Artem Serov2808be82018-12-20 19:15:11 +00001511}
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001512
Artem Serov2808be82018-12-20 19:15:11 +00001513void CodeGenerator::EmitEnvironment(HEnvironment* environment,
1514 SlowPathCode* slow_path,
1515 bool needs_vreg_info) {
1516 if (environment == nullptr) return;
1517
1518 StackMapStream* stack_map_stream = GetStackMapStream();
1519 bool emit_inline_info = environment->GetParent() != nullptr;
1520
1521 if (emit_inline_info) {
1522 // We emit the parent environment first.
1523 EmitEnvironment(environment->GetParent(), slow_path, needs_vreg_info);
1524 stack_map_stream->BeginInlineInfoEntry(environment->GetMethod(),
1525 environment->GetDexPc(),
1526 needs_vreg_info ? environment->Size() : 0,
1527 &graph_->GetDexFile());
1528 }
1529
1530 if (needs_vreg_info) {
1531 // If a dex register map is not required we just won't emit it.
1532 EmitVRegInfo(environment, slow_path);
1533 }
1534
1535 if (emit_inline_info) {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001536 stack_map_stream->EndInlineInfoEntry();
Nicolas Geoffrayb1d0f3f2015-05-14 12:41:51 +01001537 }
Nicolas Geoffray39468442014-09-02 15:17:15 +01001538}
1539
Calin Juravle77520bc2015-01-12 18:45:46 +00001540bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) {
Nicolas Geoffray61ba8d22018-08-07 09:55:57 +01001541 return null_check->IsEmittedAtUseSite();
Calin Juravle77520bc2015-01-12 18:45:46 +00001542}
1543
1544void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) {
Nicolas Geoffray61ba8d22018-08-07 09:55:57 +01001545 HNullCheck* null_check = instr->GetImplicitNullCheck();
1546 if (null_check != nullptr) {
Evgeny Astigeevich98416bf2019-09-09 14:52:12 +01001547 RecordPcInfo(null_check, null_check->GetDexPc(), GetAssembler()->CodePosition());
Calin Juravle77520bc2015-01-12 18:45:46 +00001548 }
1549}
1550
Vladimir Marko804b03f2016-09-14 16:26:36 +01001551LocationSummary* CodeGenerator::CreateThrowingSlowPathLocations(HInstruction* instruction,
1552 RegisterSet caller_saves) {
Vladimir Marko3b7537b2016-09-13 11:56:01 +00001553 // Note: Using kNoCall allows the method to be treated as leaf (and eliminate the
1554 // HSuspendCheck from entry block). However, it will still get a valid stack frame
1555 // because the HNullCheck needs an environment.
1556 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
1557 // When throwing from a try block, we may need to retrieve dalvik registers from
1558 // physical registers and we also need to set up stack mask for GC. This is
1559 // implicitly achieved by passing kCallOnSlowPath to the LocationSummary.
Vladimir Marko804b03f2016-09-14 16:26:36 +01001560 bool can_throw_into_catch_block = instruction->CanThrowIntoCatchBlock();
Vladimir Marko3b7537b2016-09-13 11:56:01 +00001561 if (can_throw_into_catch_block) {
1562 call_kind = LocationSummary::kCallOnSlowPath;
1563 }
Vladimir Markoca6fff82017-10-03 14:49:14 +01001564 LocationSummary* locations =
1565 new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind);
Vladimir Marko3b7537b2016-09-13 11:56:01 +00001566 if (can_throw_into_catch_block && compiler_options_.GetImplicitNullChecks()) {
Vladimir Marko804b03f2016-09-14 16:26:36 +01001567 locations->SetCustomSlowPathCallerSaves(caller_saves); // Default: no caller-save registers.
Vladimir Marko3b7537b2016-09-13 11:56:01 +00001568 }
Vladimir Marko804b03f2016-09-14 16:26:36 +01001569 DCHECK(!instruction->HasUses());
Vladimir Marko3b7537b2016-09-13 11:56:01 +00001570 return locations;
1571}
1572
Calin Juravle2ae48182016-03-16 14:05:09 +00001573void CodeGenerator::GenerateNullCheck(HNullCheck* instruction) {
Vladimir Marko3b7537b2016-09-13 11:56:01 +00001574 if (compiler_options_.GetImplicitNullChecks()) {
Vladimir Markocd09e1f2017-11-24 15:02:40 +00001575 MaybeRecordStat(stats_, MethodCompilationStat::kImplicitNullCheckGenerated);
Calin Juravle2ae48182016-03-16 14:05:09 +00001576 GenerateImplicitNullCheck(instruction);
1577 } else {
Vladimir Markocd09e1f2017-11-24 15:02:40 +00001578 MaybeRecordStat(stats_, MethodCompilationStat::kExplicitNullCheckGenerated);
Calin Juravle2ae48182016-03-16 14:05:09 +00001579 GenerateExplicitNullCheck(instruction);
1580 }
1581}
1582
Vladimir Markobea75ff2017-10-11 20:39:54 +01001583void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check,
1584 HParallelMove* spills) const {
Nicolas Geoffray3c049742014-09-24 18:10:46 +01001585 LocationSummary* locations = suspend_check->GetLocations();
1586 HBasicBlock* block = suspend_check->GetBlock();
1587 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check);
1588 DCHECK(block->IsLoopHeader());
Vladimir Markobea75ff2017-10-11 20:39:54 +01001589 DCHECK(block->GetFirstInstruction() == spills);
Nicolas Geoffray3c049742014-09-24 18:10:46 +01001590
Vladimir Markobea75ff2017-10-11 20:39:54 +01001591 for (size_t i = 0, num_moves = spills->NumMoves(); i != num_moves; ++i) {
1592 Location dest = spills->MoveOperandsAt(i)->GetDestination();
1593 // All parallel moves in loop headers are spills.
1594 DCHECK(dest.IsStackSlot() || dest.IsDoubleStackSlot() || dest.IsSIMDStackSlot()) << dest;
1595 // Clear the stack bit marking a reference. Do not bother to check if the spill is
1596 // actually a reference spill, clearing bits that are already zero is harmless.
1597 locations->ClearStackBit(dest.GetStackIndex() / kVRegSize);
Nicolas Geoffray3c049742014-09-24 18:10:46 +01001598 }
1599}
1600
Nicolas Geoffray90218252015-04-15 11:56:51 +01001601void CodeGenerator::EmitParallelMoves(Location from1,
1602 Location to1,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001603 DataType::Type type1,
Nicolas Geoffray90218252015-04-15 11:56:51 +01001604 Location from2,
1605 Location to2,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001606 DataType::Type type2) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001607 HParallelMove parallel_move(GetGraph()->GetAllocator());
Nicolas Geoffray90218252015-04-15 11:56:51 +01001608 parallel_move.AddMove(from1, to1, type1, nullptr);
1609 parallel_move.AddMove(from2, to2, type2, nullptr);
Nicolas Geoffrayf0e39372014-11-12 17:50:07 +00001610 GetMoveResolver()->EmitNativeCode(&parallel_move);
1611}
1612
Alexandre Rames91a65162016-09-19 13:54:30 +01001613void CodeGenerator::ValidateInvokeRuntime(QuickEntrypointEnum entrypoint,
1614 HInstruction* instruction,
1615 SlowPathCode* slow_path) {
Alexandre Rames78e3ef62015-08-12 13:43:29 +01001616 // Ensure that the call kind indication given to the register allocator is
Alexandre Rames91a65162016-09-19 13:54:30 +01001617 // coherent with the runtime call generated.
Alexandre Rames78e3ef62015-08-12 13:43:29 +01001618 if (slow_path == nullptr) {
Roland Levillain0d5a2812015-11-13 10:07:31 +00001619 DCHECK(instruction->GetLocations()->WillCall())
1620 << "instruction->DebugName()=" << instruction->DebugName();
Alexandre Rames78e3ef62015-08-12 13:43:29 +01001621 } else {
Serban Constantinescu806f0122016-03-09 11:10:16 +00001622 DCHECK(instruction->GetLocations()->CallsOnSlowPath() || slow_path->IsFatal())
Roland Levillain0d5a2812015-11-13 10:07:31 +00001623 << "instruction->DebugName()=" << instruction->DebugName()
1624 << " slow_path->GetDescription()=" << slow_path->GetDescription();
Alexandre Rames91a65162016-09-19 13:54:30 +01001625 }
1626
1627 // Check that the GC side effect is set when required.
1628 // TODO: Reverse EntrypointCanTriggerGC
1629 if (EntrypointCanTriggerGC(entrypoint)) {
1630 if (slow_path == nullptr) {
1631 DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
1632 << "instruction->DebugName()=" << instruction->DebugName()
1633 << " instruction->GetSideEffects().ToString()="
1634 << instruction->GetSideEffects().ToString();
1635 } else {
Artem Serovd1aa7d02018-06-22 11:35:46 +01001636 // 'CanTriggerGC' side effect is used to restrict optimization of instructions which depend
1637 // on GC (e.g. IntermediateAddress) - to ensure they are not alive across GC points. However
1638 // if execution never returns to the compiled code from a GC point this restriction is
1639 // unnecessary - in particular for fatal slow paths which might trigger GC.
1640 DCHECK((slow_path->IsFatal() && !instruction->GetLocations()->WillCall()) ||
1641 instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
Alexandre Rames91a65162016-09-19 13:54:30 +01001642 // When (non-Baker) read barriers are enabled, some instructions
1643 // use a slow path to emit a read barrier, which does not trigger
1644 // GC.
1645 (kEmitCompilerReadBarrier &&
1646 !kUseBakerReadBarrier &&
1647 (instruction->IsInstanceFieldGet() ||
1648 instruction->IsStaticFieldGet() ||
1649 instruction->IsArrayGet() ||
1650 instruction->IsLoadClass() ||
1651 instruction->IsLoadString() ||
1652 instruction->IsInstanceOf() ||
1653 instruction->IsCheckCast() ||
1654 (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified()))))
1655 << "instruction->DebugName()=" << instruction->DebugName()
1656 << " instruction->GetSideEffects().ToString()="
1657 << instruction->GetSideEffects().ToString()
1658 << " slow_path->GetDescription()=" << slow_path->GetDescription();
1659 }
1660 } else {
1661 // The GC side effect is not required for the instruction. But the instruction might still have
1662 // it, for example if it calls other entrypoints requiring it.
Alexandre Rames78e3ef62015-08-12 13:43:29 +01001663 }
1664
1665 // Check the coherency of leaf information.
1666 DCHECK(instruction->IsSuspendCheck()
1667 || ((slow_path != nullptr) && slow_path->IsFatal())
1668 || instruction->GetLocations()->CanCall()
Roland Levillaindf3f8222015-08-13 12:31:44 +01001669 || !IsLeafMethod())
1670 << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : "");
Alexandre Rames78e3ef62015-08-12 13:43:29 +01001671}
1672
Roland Levillaindec8f632016-07-22 17:10:06 +01001673void CodeGenerator::ValidateInvokeRuntimeWithoutRecordingPcInfo(HInstruction* instruction,
1674 SlowPathCode* slow_path) {
1675 DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath())
1676 << "instruction->DebugName()=" << instruction->DebugName()
1677 << " slow_path->GetDescription()=" << slow_path->GetDescription();
1678 // Only the Baker read barrier marking slow path used by certains
1679 // instructions is expected to invoke the runtime without recording
1680 // PC-related information.
1681 DCHECK(kUseBakerReadBarrier);
1682 DCHECK(instruction->IsInstanceFieldGet() ||
1683 instruction->IsStaticFieldGet() ||
1684 instruction->IsArrayGet() ||
Roland Levillain16d9f942016-08-25 17:27:56 +01001685 instruction->IsArraySet() ||
Roland Levillaindec8f632016-07-22 17:10:06 +01001686 instruction->IsLoadClass() ||
1687 instruction->IsLoadString() ||
1688 instruction->IsInstanceOf() ||
1689 instruction->IsCheckCast() ||
Roland Levillain0b671c02016-08-19 12:02:34 +01001690 (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified()) ||
1691 (instruction->IsInvokeStaticOrDirect() && instruction->GetLocations()->Intrinsified()))
Roland Levillaindec8f632016-07-22 17:10:06 +01001692 << "instruction->DebugName()=" << instruction->DebugName()
1693 << " slow_path->GetDescription()=" << slow_path->GetDescription();
1694}
1695
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001696void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001697 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
Roland Levillain0d5a2812015-11-13 10:07:31 +00001698
Andreas Gampe3db70682018-12-26 15:12:03 -08001699 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers= */ true);
Vladimir Marko70e97462016-08-09 11:04:26 +01001700 for (uint32_t i : LowToHighBits(core_spills)) {
1701 // If the register holds an object, update the stack mask.
1702 if (locations->RegisterContainsObject(i)) {
1703 locations->SetStackBit(stack_offset / kVRegSize);
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001704 }
Vladimir Marko70e97462016-08-09 11:04:26 +01001705 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1706 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1707 saved_core_stack_offsets_[i] = stack_offset;
1708 stack_offset += codegen->SaveCoreRegister(stack_offset, i);
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001709 }
1710
Andreas Gampe3db70682018-12-26 15:12:03 -08001711 const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers= */ false);
Vladimir Marko804b03f2016-09-14 16:26:36 +01001712 for (uint32_t i : LowToHighBits(fp_spills)) {
Vladimir Marko70e97462016-08-09 11:04:26 +01001713 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1714 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1715 saved_fpu_stack_offsets_[i] = stack_offset;
1716 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i);
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001717 }
1718}
1719
1720void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001721 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
Roland Levillain0d5a2812015-11-13 10:07:31 +00001722
Andreas Gampe3db70682018-12-26 15:12:03 -08001723 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers= */ true);
Vladimir Marko70e97462016-08-09 11:04:26 +01001724 for (uint32_t i : LowToHighBits(core_spills)) {
1725 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1726 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1727 stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001728 }
1729
Andreas Gampe3db70682018-12-26 15:12:03 -08001730 const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers= */ false);
Vladimir Marko804b03f2016-09-14 16:26:36 +01001731 for (uint32_t i : LowToHighBits(fp_spills)) {
Vladimir Marko70e97462016-08-09 11:04:26 +01001732 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1733 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1734 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00001735 }
1736}
1737
Nicolas Geoffray5bd05a52015-10-13 09:48:30 +01001738void CodeGenerator::CreateSystemArrayCopyLocationSummary(HInvoke* invoke) {
1739 // Check to see if we have known failures that will cause us to have to bail out
1740 // to the runtime, and just generate the runtime call directly.
1741 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1742 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
1743
1744 // The positions must be non-negative.
1745 if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
1746 (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
1747 // We will have to fail anyways.
1748 return;
1749 }
1750
1751 // The length must be >= 0.
1752 HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1753 if (length != nullptr) {
1754 int32_t len = length->GetValue();
1755 if (len < 0) {
1756 // Just call as normal.
1757 return;
1758 }
1759 }
1760
1761 SystemArrayCopyOptimizations optimizations(invoke);
1762
1763 if (optimizations.GetDestinationIsSource()) {
1764 if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
1765 // We only support backward copying if source and destination are the same.
1766 return;
1767 }
1768 }
1769
1770 if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
1771 // We currently don't intrinsify primitive copying.
1772 return;
1773 }
1774
Vladimir Markoca6fff82017-10-03 14:49:14 +01001775 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator();
Nicolas Geoffray5bd05a52015-10-13 09:48:30 +01001776 LocationSummary* locations = new (allocator) LocationSummary(invoke,
1777 LocationSummary::kCallOnSlowPath,
1778 kIntrinsified);
1779 // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
1780 locations->SetInAt(0, Location::RequiresRegister());
1781 locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
1782 locations->SetInAt(2, Location::RequiresRegister());
1783 locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3)));
1784 locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4)));
1785
1786 locations->AddTemp(Location::RequiresRegister());
1787 locations->AddTemp(Location::RequiresRegister());
1788 locations->AddTemp(Location::RequiresRegister());
1789}
1790
Nicolas Geoffray132d8362016-11-16 09:19:42 +00001791void CodeGenerator::EmitJitRoots(uint8_t* code,
Vladimir Markoac3ac682018-09-20 11:01:43 +01001792 const uint8_t* roots_data,
1793 /*out*/std::vector<Handle<mirror::Object>>* roots) {
Vladimir Marko174b2e22017-10-12 13:34:49 +01001794 code_generation_data_->EmitJitRoots(roots);
Nicolas Geoffray132d8362016-11-16 09:19:42 +00001795 EmitJitRootPatches(code, roots_data);
1796}
1797
Vladimir Markob5461632018-10-15 14:24:21 +01001798QuickEntrypointEnum CodeGenerator::GetArrayAllocationEntrypoint(HNewArray* new_array) {
1799 switch (new_array->GetComponentSizeShift()) {
1800 case 0: return kQuickAllocArrayResolved8;
1801 case 1: return kQuickAllocArrayResolved16;
1802 case 2: return kQuickAllocArrayResolved32;
1803 case 3: return kQuickAllocArrayResolved64;
Nicolas Geoffrayb048cb72017-01-23 22:50:24 +00001804 }
1805 LOG(FATAL) << "Unreachable";
Elliott Hughesc1896c92018-11-29 11:33:18 -08001806 UNREACHABLE();
Nicolas Geoffrayb048cb72017-01-23 22:50:24 +00001807}
1808
Nicolas Geoffrayd4dd2552014-02-28 10:23:58 +00001809} // namespace art