blob: 5691dd0d4a26fa81da8cc5da381d453cf3ede353 [file] [log] [blame]
Andreas Gampe878d58c2015-01-15 23:24:00 -08001/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
Serban Constantinescu579885a2015-02-22 20:51:33 +000019#include "arch/arm64/instruction_set_features_arm64.h"
Mathieu Chartiere401d142015-04-22 13:56:20 -070020#include "art_method.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080021#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080025#include "lock_word.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080026#include "mirror/array-inl.h"
Andreas Gampec15a2f42017-04-21 12:09:39 -070027#include "mirror/object_array-inl.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080028#include "mirror/reference.h"
Vladimir Markoe39f14f2017-02-10 15:44:25 +000029#include "mirror/string-inl.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080030#include "scoped_thread_state_change-inl.h"
Andreas Gampeb486a982017-06-01 13:45:54 -070031#include "thread-current-inl.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080032#include "utils/arm64/assembler_arm64.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080033
Scott Wakeling97c72b72016-06-24 16:19:36 +010034using namespace vixl::aarch64; // NOLINT(build/namespaces)
Andreas Gampe878d58c2015-01-15 23:24:00 -080035
Artem Serovaf4e42a2016-08-08 15:11:24 +010036// TODO(VIXL): Make VIXL compile with -Wshadow.
Scott Wakeling97c72b72016-06-24 16:19:36 +010037#pragma GCC diagnostic push
38#pragma GCC diagnostic ignored "-Wshadow"
Artem Serovaf4e42a2016-08-08 15:11:24 +010039#include "aarch64/disasm-aarch64.h"
40#include "aarch64/macro-assembler-aarch64.h"
Scott Wakeling97c72b72016-06-24 16:19:36 +010041#pragma GCC diagnostic pop
Andreas Gampe878d58c2015-01-15 23:24:00 -080042
43namespace art {
44
45namespace arm64 {
46
47using helpers::DRegisterFrom;
48using helpers::FPRegisterFrom;
49using helpers::HeapOperand;
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +000050using helpers::LocationFrom;
Scott Wakeling9ee23f42015-07-23 10:44:35 +010051using helpers::OperandFrom;
Andreas Gampe878d58c2015-01-15 23:24:00 -080052using helpers::RegisterFrom;
53using helpers::SRegisterFrom;
54using helpers::WRegisterFrom;
55using helpers::XRegisterFrom;
xueliang.zhong49924c92016-03-03 10:52:51 +000056using helpers::InputRegisterAt;
Scott Wakeling1f36f412016-04-21 11:13:45 +010057using helpers::OutputRegister;
Andreas Gampe878d58c2015-01-15 23:24:00 -080058
Andreas Gampe878d58c2015-01-15 23:24:00 -080059namespace {
60
61ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
62 return MemOperand(XRegisterFrom(location), offset);
63}
64
65} // namespace
66
Scott Wakeling97c72b72016-06-24 16:19:36 +010067MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
Alexandre Rames087930f2016-08-02 13:45:28 +010068 return codegen_->GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -080069}
70
71ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
72 return codegen_->GetGraph()->GetArena();
73}
74
Alexandre Rames087930f2016-08-02 13:45:28 +010075#define __ codegen->GetVIXLAssembler()->
Andreas Gampe878d58c2015-01-15 23:24:00 -080076
77static void MoveFromReturnRegister(Location trg,
78 Primitive::Type type,
79 CodeGeneratorARM64* codegen) {
80 if (!trg.IsValid()) {
81 DCHECK(type == Primitive::kPrimVoid);
82 return;
83 }
84
85 DCHECK_NE(type, Primitive::kPrimVoid);
86
Jeff Hao848f70a2014-01-15 13:49:50 -080087 if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
Andreas Gampe878d58c2015-01-15 23:24:00 -080088 Register trg_reg = RegisterFrom(trg, type);
89 Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
90 __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
91 } else {
92 FPRegister trg_reg = FPRegisterFrom(trg, type);
93 FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
94 __ Fmov(trg_reg, res_reg);
95 }
96}
97
Roland Levillainec525fc2015-04-28 15:50:20 +010098static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
Roland Levillain2d27c8e2015-04-28 15:48:45 +010099 InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
Roland Levillainec525fc2015-04-28 15:50:20 +0100100 IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800101}
102
103// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
104// call. This will copy the arguments into the positions for a regular call.
105//
106// Note: The actual parameters are required to be in the locations given by the invoke's location
107// summary. If an intrinsic modifies those locations before a slowpath call, they must be
108// restored!
109class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
110 public:
David Srbecky9cd6d372016-02-09 15:24:47 +0000111 explicit IntrinsicSlowPathARM64(HInvoke* invoke)
112 : SlowPathCodeARM64(invoke), invoke_(invoke) { }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800113
114 void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
115 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
116 __ Bind(GetEntryLabel());
117
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000118 SaveLiveRegisters(codegen, invoke_->GetLocations());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800119
Roland Levillainec525fc2015-04-28 15:50:20 +0100120 MoveArguments(invoke_, codegen);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800121
Artem Serov914d7a82017-02-07 14:33:49 +0000122 {
123 // Ensure that between the BLR (emitted by Generate*Call) and RecordPcInfo there
124 // are no pools emitted.
125 vixl::EmissionCheckScope guard(codegen->GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
126 if (invoke_->IsInvokeStaticOrDirect()) {
Vladimir Markoe7197bf2017-06-02 17:00:23 +0100127 codegen->GenerateStaticOrDirectCall(
128 invoke_->AsInvokeStaticOrDirect(), LocationFrom(kArtMethodRegister), this);
Artem Serov914d7a82017-02-07 14:33:49 +0000129 } else {
Vladimir Markoe7197bf2017-06-02 17:00:23 +0100130 codegen->GenerateVirtualCall(
131 invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister), this);
Artem Serov914d7a82017-02-07 14:33:49 +0000132 }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800133 }
134
135 // Copy the result back to the expected output.
136 Location out = invoke_->GetLocations()->Out();
137 if (out.IsValid()) {
138 DCHECK(out.IsRegister()); // TODO: Replace this when we support output in memory.
139 DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
140 MoveFromReturnRegister(out, invoke_->GetType(), codegen);
141 }
142
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000143 RestoreLiveRegisters(codegen, invoke_->GetLocations());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800144 __ B(GetExitLabel());
145 }
146
Alexandre Rames9931f312015-06-19 14:47:01 +0100147 const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
148
Andreas Gampe878d58c2015-01-15 23:24:00 -0800149 private:
150 // The instruction where this slow path is happening.
151 HInvoke* const invoke_;
152
153 DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
154};
155
Roland Levillain0b671c02016-08-19 12:02:34 +0100156// Slow path implementing the SystemArrayCopy intrinsic copy loop with read barriers.
157class ReadBarrierSystemArrayCopySlowPathARM64 : public SlowPathCodeARM64 {
158 public:
159 ReadBarrierSystemArrayCopySlowPathARM64(HInstruction* instruction, Location tmp)
160 : SlowPathCodeARM64(instruction), tmp_(tmp) {
161 DCHECK(kEmitCompilerReadBarrier);
162 DCHECK(kUseBakerReadBarrier);
163 }
164
165 void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
166 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
167 LocationSummary* locations = instruction_->GetLocations();
168 DCHECK(locations->CanCall());
169 DCHECK(instruction_->IsInvokeStaticOrDirect())
170 << "Unexpected instruction in read barrier arraycopy slow path: "
171 << instruction_->DebugName();
172 DCHECK(instruction_->GetLocations()->Intrinsified());
173 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy);
174
175 const int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
176
177 Register src_curr_addr = XRegisterFrom(locations->GetTemp(0));
178 Register dst_curr_addr = XRegisterFrom(locations->GetTemp(1));
179 Register src_stop_addr = XRegisterFrom(locations->GetTemp(2));
180 Register tmp_reg = WRegisterFrom(tmp_);
181
182 __ Bind(GetEntryLabel());
183 vixl::aarch64::Label slow_copy_loop;
184 __ Bind(&slow_copy_loop);
185 __ Ldr(tmp_reg, MemOperand(src_curr_addr, element_size, PostIndex));
186 codegen->GetAssembler()->MaybeUnpoisonHeapReference(tmp_reg);
187 // TODO: Inline the mark bit check before calling the runtime?
188 // tmp_reg = ReadBarrier::Mark(tmp_reg);
189 // No need to save live registers; it's taken care of by the
190 // entrypoint. Also, there is no need to update the stack mask,
191 // as this runtime call will not trigger a garbage collection.
192 // (See ReadBarrierMarkSlowPathARM64::EmitNativeCode for more
193 // explanations.)
194 DCHECK_NE(tmp_.reg(), LR);
195 DCHECK_NE(tmp_.reg(), WSP);
196 DCHECK_NE(tmp_.reg(), WZR);
197 // IP0 is used internally by the ReadBarrierMarkRegX entry point
198 // as a temporary (and not preserved). It thus cannot be used by
199 // any live register in this slow path.
200 DCHECK_NE(LocationFrom(src_curr_addr).reg(), IP0);
201 DCHECK_NE(LocationFrom(dst_curr_addr).reg(), IP0);
202 DCHECK_NE(LocationFrom(src_stop_addr).reg(), IP0);
203 DCHECK_NE(tmp_.reg(), IP0);
204 DCHECK(0 <= tmp_.reg() && tmp_.reg() < kNumberOfWRegisters) << tmp_.reg();
Roland Levillain9cc0ea82017-03-16 11:25:59 +0000205 // TODO: Load the entrypoint once before the loop, instead of
206 // loading it at every iteration.
Roland Levillain0b671c02016-08-19 12:02:34 +0100207 int32_t entry_point_offset =
Roland Levillain97c46462017-05-11 14:04:03 +0100208 Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg());
Roland Levillain0b671c02016-08-19 12:02:34 +0100209 // This runtime call does not require a stack map.
210 codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
211 codegen->GetAssembler()->MaybePoisonHeapReference(tmp_reg);
212 __ Str(tmp_reg, MemOperand(dst_curr_addr, element_size, PostIndex));
213 __ Cmp(src_curr_addr, src_stop_addr);
214 __ B(&slow_copy_loop, ne);
215 __ B(GetExitLabel());
216 }
217
218 const char* GetDescription() const OVERRIDE { return "ReadBarrierSystemArrayCopySlowPathARM64"; }
219
220 private:
221 Location tmp_;
222
223 DISALLOW_COPY_AND_ASSIGN(ReadBarrierSystemArrayCopySlowPathARM64);
224};
Andreas Gampe878d58c2015-01-15 23:24:00 -0800225#undef __
226
227bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
228 Dispatch(invoke);
229 LocationSummary* res = invoke->GetLocations();
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000230 if (res == nullptr) {
231 return false;
232 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000233 return res->Intrinsified();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800234}
235
236#define __ masm->
237
238static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
239 LocationSummary* locations = new (arena) LocationSummary(invoke,
240 LocationSummary::kNoCall,
241 kIntrinsified);
242 locations->SetInAt(0, Location::RequiresFpuRegister());
243 locations->SetOut(Location::RequiresRegister());
244}
245
246static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
247 LocationSummary* locations = new (arena) LocationSummary(invoke,
248 LocationSummary::kNoCall,
249 kIntrinsified);
250 locations->SetInAt(0, Location::RequiresRegister());
251 locations->SetOut(Location::RequiresFpuRegister());
252}
253
Scott Wakeling97c72b72016-06-24 16:19:36 +0100254static void MoveFPToInt(LocationSummary* locations, bool is64bit, MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800255 Location input = locations->InAt(0);
256 Location output = locations->Out();
257 __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
258 is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
259}
260
Scott Wakeling97c72b72016-06-24 16:19:36 +0100261static void MoveIntToFP(LocationSummary* locations, bool is64bit, MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800262 Location input = locations->InAt(0);
263 Location output = locations->Out();
264 __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
265 is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
266}
267
268void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
269 CreateFPToIntLocations(arena_, invoke);
270}
271void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
272 CreateIntToFPLocations(arena_, invoke);
273}
274
275void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000276 MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800277}
278void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000279 MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800280}
281
282void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
283 CreateFPToIntLocations(arena_, invoke);
284}
285void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
286 CreateIntToFPLocations(arena_, invoke);
287}
288
289void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000290 MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800291}
292void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000293 MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800294}
295
296static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
297 LocationSummary* locations = new (arena) LocationSummary(invoke,
298 LocationSummary::kNoCall,
299 kIntrinsified);
300 locations->SetInAt(0, Location::RequiresRegister());
301 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
302}
303
304static void GenReverseBytes(LocationSummary* locations,
305 Primitive::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100306 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800307 Location in = locations->InAt(0);
308 Location out = locations->Out();
309
310 switch (type) {
311 case Primitive::kPrimShort:
312 __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
313 __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
314 break;
315 case Primitive::kPrimInt:
316 case Primitive::kPrimLong:
317 __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
318 break;
319 default:
320 LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
321 UNREACHABLE();
322 }
323}
324
325void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
326 CreateIntToIntLocations(arena_, invoke);
327}
328
329void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
330 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
331}
332
333void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
334 CreateIntToIntLocations(arena_, invoke);
335}
336
337void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
338 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
339}
340
341void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
342 CreateIntToIntLocations(arena_, invoke);
343}
344
345void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
346 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
347}
348
Aart Bik7b565022016-01-28 14:36:22 -0800349static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
350 LocationSummary* locations = new (arena) LocationSummary(invoke,
351 LocationSummary::kNoCall,
352 kIntrinsified);
353 locations->SetInAt(0, Location::RequiresRegister());
354 locations->SetInAt(1, Location::RequiresRegister());
355 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
356}
357
Scott Wakeling611d3392015-07-10 11:42:06 +0100358static void GenNumberOfLeadingZeros(LocationSummary* locations,
359 Primitive::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100360 MacroAssembler* masm) {
Scott Wakeling611d3392015-07-10 11:42:06 +0100361 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
362
363 Location in = locations->InAt(0);
364 Location out = locations->Out();
365
366 __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
367}
368
369void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
370 CreateIntToIntLocations(arena_, invoke);
371}
372
373void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
374 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
375}
376
377void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
378 CreateIntToIntLocations(arena_, invoke);
379}
380
381void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
382 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
383}
384
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100385static void GenNumberOfTrailingZeros(LocationSummary* locations,
386 Primitive::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100387 MacroAssembler* masm) {
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100388 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
389
390 Location in = locations->InAt(0);
391 Location out = locations->Out();
392
393 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
394 __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
395}
396
397void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
398 CreateIntToIntLocations(arena_, invoke);
399}
400
401void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
402 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
403}
404
405void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
406 CreateIntToIntLocations(arena_, invoke);
407}
408
409void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
410 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
411}
412
Andreas Gampe878d58c2015-01-15 23:24:00 -0800413static void GenReverse(LocationSummary* locations,
414 Primitive::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100415 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800416 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
417
418 Location in = locations->InAt(0);
419 Location out = locations->Out();
420
421 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
422}
423
424void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
425 CreateIntToIntLocations(arena_, invoke);
426}
427
428void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
429 GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
430}
431
432void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
433 CreateIntToIntLocations(arena_, invoke);
434}
435
436void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
437 GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
438}
439
Scott Wakeling97c72b72016-06-24 16:19:36 +0100440static void GenBitCount(HInvoke* instr, Primitive::Type type, MacroAssembler* masm) {
Roland Levillainfa3912e2016-04-01 18:21:55 +0100441 DCHECK(Primitive::IsIntOrLongType(type)) << type;
442 DCHECK_EQ(instr->GetType(), Primitive::kPrimInt);
443 DCHECK_EQ(Primitive::PrimitiveKind(instr->InputAt(0)->GetType()), type);
xueliang.zhong49924c92016-03-03 10:52:51 +0000444
xueliang.zhong49924c92016-03-03 10:52:51 +0000445 UseScratchRegisterScope temps(masm);
446
Nicolas Geoffray457413a2016-03-04 11:10:17 +0000447 Register src = InputRegisterAt(instr, 0);
Roland Levillainfa3912e2016-04-01 18:21:55 +0100448 Register dst = RegisterFrom(instr->GetLocations()->Out(), type);
449 FPRegister fpr = (type == Primitive::kPrimLong) ? temps.AcquireD() : temps.AcquireS();
xueliang.zhong49924c92016-03-03 10:52:51 +0000450
451 __ Fmov(fpr, src);
Nicolas Geoffray457413a2016-03-04 11:10:17 +0000452 __ Cnt(fpr.V8B(), fpr.V8B());
453 __ Addv(fpr.B(), fpr.V8B());
xueliang.zhong49924c92016-03-03 10:52:51 +0000454 __ Fmov(dst, fpr);
455}
456
457void IntrinsicLocationsBuilderARM64::VisitLongBitCount(HInvoke* invoke) {
458 CreateIntToIntLocations(arena_, invoke);
459}
460
461void IntrinsicCodeGeneratorARM64::VisitLongBitCount(HInvoke* invoke) {
Roland Levillainfa3912e2016-04-01 18:21:55 +0100462 GenBitCount(invoke, Primitive::kPrimLong, GetVIXLAssembler());
xueliang.zhong49924c92016-03-03 10:52:51 +0000463}
464
465void IntrinsicLocationsBuilderARM64::VisitIntegerBitCount(HInvoke* invoke) {
466 CreateIntToIntLocations(arena_, invoke);
467}
468
469void IntrinsicCodeGeneratorARM64::VisitIntegerBitCount(HInvoke* invoke) {
Roland Levillainfa3912e2016-04-01 18:21:55 +0100470 GenBitCount(invoke, Primitive::kPrimInt, GetVIXLAssembler());
xueliang.zhong49924c92016-03-03 10:52:51 +0000471}
472
Andreas Gampe878d58c2015-01-15 23:24:00 -0800473static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800474 LocationSummary* locations = new (arena) LocationSummary(invoke,
475 LocationSummary::kNoCall,
476 kIntrinsified);
477 locations->SetInAt(0, Location::RequiresFpuRegister());
478 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
479}
480
Scott Wakeling97c72b72016-06-24 16:19:36 +0100481static void MathAbsFP(LocationSummary* locations, bool is64bit, MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800482 Location in = locations->InAt(0);
483 Location out = locations->Out();
484
485 FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
486 FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
487
488 __ Fabs(out_reg, in_reg);
489}
490
491void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
492 CreateFPToFPLocations(arena_, invoke);
493}
494
495void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000496 MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800497}
498
499void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
500 CreateFPToFPLocations(arena_, invoke);
501}
502
503void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000504 MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800505}
506
507static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
508 LocationSummary* locations = new (arena) LocationSummary(invoke,
509 LocationSummary::kNoCall,
510 kIntrinsified);
511 locations->SetInAt(0, Location::RequiresRegister());
512 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
513}
514
515static void GenAbsInteger(LocationSummary* locations,
516 bool is64bit,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100517 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800518 Location in = locations->InAt(0);
519 Location output = locations->Out();
520
521 Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
522 Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
523
524 __ Cmp(in_reg, Operand(0));
525 __ Cneg(out_reg, in_reg, lt);
526}
527
528void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
529 CreateIntToInt(arena_, invoke);
530}
531
532void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000533 GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800534}
535
536void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
537 CreateIntToInt(arena_, invoke);
538}
539
540void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000541 GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800542}
543
544static void GenMinMaxFP(LocationSummary* locations,
545 bool is_min,
546 bool is_double,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100547 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800548 Location op1 = locations->InAt(0);
549 Location op2 = locations->InAt(1);
550 Location out = locations->Out();
551
552 FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
553 FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
554 FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
555 if (is_min) {
556 __ Fmin(out_reg, op1_reg, op2_reg);
557 } else {
558 __ Fmax(out_reg, op1_reg, op2_reg);
559 }
560}
561
562static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
563 LocationSummary* locations = new (arena) LocationSummary(invoke,
564 LocationSummary::kNoCall,
565 kIntrinsified);
566 locations->SetInAt(0, Location::RequiresFpuRegister());
567 locations->SetInAt(1, Location::RequiresFpuRegister());
568 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
569}
570
571void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
572 CreateFPFPToFPLocations(arena_, invoke);
573}
574
575void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000576 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800577}
578
579void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
580 CreateFPFPToFPLocations(arena_, invoke);
581}
582
583void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000584 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800585}
586
587void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
588 CreateFPFPToFPLocations(arena_, invoke);
589}
590
591void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000592 GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800593}
594
595void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
596 CreateFPFPToFPLocations(arena_, invoke);
597}
598
599void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000600 GenMinMaxFP(
601 invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800602}
603
604static void GenMinMax(LocationSummary* locations,
605 bool is_min,
606 bool is_long,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100607 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800608 Location op1 = locations->InAt(0);
609 Location op2 = locations->InAt(1);
610 Location out = locations->Out();
611
612 Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
613 Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
614 Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
615
616 __ Cmp(op1_reg, op2_reg);
617 __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
618}
619
Andreas Gampe878d58c2015-01-15 23:24:00 -0800620void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
621 CreateIntIntToIntLocations(arena_, invoke);
622}
623
624void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000625 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800626}
627
628void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
629 CreateIntIntToIntLocations(arena_, invoke);
630}
631
632void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000633 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800634}
635
636void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
637 CreateIntIntToIntLocations(arena_, invoke);
638}
639
640void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000641 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800642}
643
644void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
645 CreateIntIntToIntLocations(arena_, invoke);
646}
647
648void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000649 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800650}
651
652void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
653 CreateFPToFPLocations(arena_, invoke);
654}
655
656void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
657 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100658 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800659 __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
660}
661
662void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
663 CreateFPToFPLocations(arena_, invoke);
664}
665
666void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
667 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100668 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800669 __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
670}
671
672void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
673 CreateFPToFPLocations(arena_, invoke);
674}
675
676void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
677 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100678 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800679 __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
680}
681
682void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
683 CreateFPToFPLocations(arena_, invoke);
684}
685
686void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
687 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100688 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800689 __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
690}
691
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100692static void CreateFPToIntPlusFPTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800693 LocationSummary* locations = new (arena) LocationSummary(invoke,
694 LocationSummary::kNoCall,
695 kIntrinsified);
696 locations->SetInAt(0, Location::RequiresFpuRegister());
697 locations->SetOut(Location::RequiresRegister());
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100698 locations->AddTemp(Location::RequiresFpuRegister());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800699}
700
Scott Wakeling97c72b72016-06-24 16:19:36 +0100701static void GenMathRound(HInvoke* invoke, bool is_double, vixl::aarch64::MacroAssembler* masm) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100702 // Java 8 API definition for Math.round():
703 // Return the closest long or int to the argument, with ties rounding to positive infinity.
704 //
705 // There is no single instruction in ARMv8 that can support the above definition.
706 // We choose to use FCVTAS here, because it has closest semantic.
707 // FCVTAS performs rounding to nearest integer, ties away from zero.
708 // For most inputs (positive values, zero or NaN), this instruction is enough.
709 // We only need a few handling code after FCVTAS if the input is negative half value.
710 //
711 // The reason why we didn't choose FCVTPS instruction here is that
712 // although it performs rounding toward positive infinity, it doesn't perform rounding to nearest.
713 // For example, FCVTPS(-1.9) = -1 and FCVTPS(1.1) = 2.
714 // If we were using this instruction, for most inputs, more handling code would be needed.
715 LocationSummary* l = invoke->GetLocations();
716 FPRegister in_reg = is_double ? DRegisterFrom(l->InAt(0)) : SRegisterFrom(l->InAt(0));
717 FPRegister tmp_fp = is_double ? DRegisterFrom(l->GetTemp(0)) : SRegisterFrom(l->GetTemp(0));
718 Register out_reg = is_double ? XRegisterFrom(l->Out()) : WRegisterFrom(l->Out());
Scott Wakeling97c72b72016-06-24 16:19:36 +0100719 vixl::aarch64::Label done;
Andreas Gampe878d58c2015-01-15 23:24:00 -0800720
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100721 // Round to nearest integer, ties away from zero.
722 __ Fcvtas(out_reg, in_reg);
723
724 // For positive values, zero or NaN inputs, rounding is done.
Scott Wakeling97c72b72016-06-24 16:19:36 +0100725 __ Tbz(out_reg, out_reg.GetSizeInBits() - 1, &done);
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100726
727 // Handle input < 0 cases.
728 // If input is negative but not a tie, previous result (round to nearest) is valid.
729 // If input is a negative tie, out_reg += 1.
730 __ Frinta(tmp_fp, in_reg);
731 __ Fsub(tmp_fp, in_reg, tmp_fp);
732 __ Fcmp(tmp_fp, 0.5);
733 __ Cinc(out_reg, out_reg, eq);
734
735 __ Bind(&done);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800736}
737
738void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100739 CreateFPToIntPlusFPTempLocations(arena_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800740}
741
742void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100743 GenMathRound(invoke, /* is_double */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800744}
745
746void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100747 CreateFPToIntPlusFPTempLocations(arena_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800748}
749
750void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100751 GenMathRound(invoke, /* is_double */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800752}
753
754void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
755 CreateIntToIntLocations(arena_, invoke);
756}
757
758void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100759 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800760 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
761 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
762}
763
764void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
765 CreateIntToIntLocations(arena_, invoke);
766}
767
768void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100769 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800770 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
771 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
772}
773
774void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
775 CreateIntToIntLocations(arena_, invoke);
776}
777
778void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100779 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800780 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
781 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
782}
783
784void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
785 CreateIntToIntLocations(arena_, invoke);
786}
787
788void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100789 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800790 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
791 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
792}
793
794static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
795 LocationSummary* locations = new (arena) LocationSummary(invoke,
796 LocationSummary::kNoCall,
797 kIntrinsified);
798 locations->SetInAt(0, Location::RequiresRegister());
799 locations->SetInAt(1, Location::RequiresRegister());
800}
801
802void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
803 CreateIntIntToVoidLocations(arena_, invoke);
804}
805
806void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100807 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800808 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
809 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
810}
811
812void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
813 CreateIntIntToVoidLocations(arena_, invoke);
814}
815
816void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100817 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800818 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
819 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
820}
821
822void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
823 CreateIntIntToVoidLocations(arena_, invoke);
824}
825
826void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100827 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800828 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
829 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
830}
831
832void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
833 CreateIntIntToVoidLocations(arena_, invoke);
834}
835
836void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100837 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800838 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
839 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
840}
841
842void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
843 LocationSummary* locations = new (arena_) LocationSummary(invoke,
844 LocationSummary::kNoCall,
845 kIntrinsified);
846 locations->SetOut(Location::RequiresRegister());
847}
848
849void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
850 codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
Andreas Gampe542451c2016-07-26 09:02:02 -0700851 MemOperand(tr, Thread::PeerOffset<kArm64PointerSize>().Int32Value()));
Andreas Gampe878d58c2015-01-15 23:24:00 -0800852}
853
854static void GenUnsafeGet(HInvoke* invoke,
855 Primitive::Type type,
856 bool is_volatile,
857 CodeGeneratorARM64* codegen) {
858 LocationSummary* locations = invoke->GetLocations();
859 DCHECK((type == Primitive::kPrimInt) ||
860 (type == Primitive::kPrimLong) ||
861 (type == Primitive::kPrimNot));
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000862 Location base_loc = locations->InAt(1);
863 Register base = WRegisterFrom(base_loc); // Object pointer.
864 Location offset_loc = locations->InAt(2);
865 Register offset = XRegisterFrom(offset_loc); // Long offset.
866 Location trg_loc = locations->Out();
867 Register trg = RegisterFrom(trg_loc, type);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800868
Roland Levillain44015862016-01-22 11:47:17 +0000869 if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
870 // UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case.
Roland Levillain54f869e2017-03-06 13:54:11 +0000871 Register temp = WRegisterFrom(locations->GetTemp(0));
Roland Levillainbfea3352016-06-23 13:48:47 +0100872 codegen->GenerateReferenceLoadWithBakerReadBarrier(invoke,
873 trg_loc,
874 base,
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100875 /* offset */ 0u,
Roland Levillainbfea3352016-06-23 13:48:47 +0100876 /* index */ offset_loc,
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100877 /* scale_factor */ 0u,
Roland Levillainbfea3352016-06-23 13:48:47 +0100878 temp,
879 /* needs_null_check */ false,
880 is_volatile);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800881 } else {
Roland Levillain44015862016-01-22 11:47:17 +0000882 // Other cases.
883 MemOperand mem_op(base.X(), offset);
884 if (is_volatile) {
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +0000885 codegen->LoadAcquire(invoke, trg, mem_op, /* needs_null_check */ true);
Roland Levillain44015862016-01-22 11:47:17 +0000886 } else {
887 codegen->Load(type, trg, mem_op);
888 }
Roland Levillain4d027112015-07-01 15:41:14 +0100889
Roland Levillain44015862016-01-22 11:47:17 +0000890 if (type == Primitive::kPrimNot) {
891 DCHECK(trg.IsW());
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100892 codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0u, offset_loc);
Roland Levillain44015862016-01-22 11:47:17 +0000893 }
Roland Levillain4d027112015-07-01 15:41:14 +0100894 }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800895}
896
897static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000898 bool can_call = kEmitCompilerReadBarrier &&
899 (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
900 invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800901 LocationSummary* locations = new (arena) LocationSummary(invoke,
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100902 (can_call
903 ? LocationSummary::kCallOnSlowPath
904 : LocationSummary::kNoCall),
Andreas Gampe878d58c2015-01-15 23:24:00 -0800905 kIntrinsified);
Vladimir Marko70e97462016-08-09 11:04:26 +0100906 if (can_call && kUseBakerReadBarrier) {
Vladimir Marko804b03f2016-09-14 16:26:36 +0100907 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
Roland Levillain54f869e2017-03-06 13:54:11 +0000908 // We need a temporary register for the read barrier marking slow
909 // path in CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier.
910 locations->AddTemp(Location::RequiresRegister());
Vladimir Marko70e97462016-08-09 11:04:26 +0100911 }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800912 locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
913 locations->SetInAt(1, Location::RequiresRegister());
914 locations->SetInAt(2, Location::RequiresRegister());
Roland Levillainbfea3352016-06-23 13:48:47 +0100915 locations->SetOut(Location::RequiresRegister(),
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100916 (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap));
Andreas Gampe878d58c2015-01-15 23:24:00 -0800917}
918
919void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
920 CreateIntIntIntToIntLocations(arena_, invoke);
921}
922void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
923 CreateIntIntIntToIntLocations(arena_, invoke);
924}
925void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
926 CreateIntIntIntToIntLocations(arena_, invoke);
927}
928void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
929 CreateIntIntIntToIntLocations(arena_, invoke);
930}
931void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
932 CreateIntIntIntToIntLocations(arena_, invoke);
933}
934void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
935 CreateIntIntIntToIntLocations(arena_, invoke);
936}
937
938void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000939 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800940}
941void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000942 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800943}
944void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000945 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800946}
947void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000948 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800949}
950void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000951 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800952}
953void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000954 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800955}
956
957static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
958 LocationSummary* locations = new (arena) LocationSummary(invoke,
959 LocationSummary::kNoCall,
960 kIntrinsified);
961 locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
962 locations->SetInAt(1, Location::RequiresRegister());
963 locations->SetInAt(2, Location::RequiresRegister());
964 locations->SetInAt(3, Location::RequiresRegister());
965}
966
967void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
968 CreateIntIntIntIntToVoid(arena_, invoke);
969}
970void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
971 CreateIntIntIntIntToVoid(arena_, invoke);
972}
973void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
974 CreateIntIntIntIntToVoid(arena_, invoke);
975}
976void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
977 CreateIntIntIntIntToVoid(arena_, invoke);
978}
979void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
980 CreateIntIntIntIntToVoid(arena_, invoke);
981}
982void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
983 CreateIntIntIntIntToVoid(arena_, invoke);
984}
985void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
986 CreateIntIntIntIntToVoid(arena_, invoke);
987}
988void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
989 CreateIntIntIntIntToVoid(arena_, invoke);
990}
991void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
992 CreateIntIntIntIntToVoid(arena_, invoke);
993}
994
Artem Serov914d7a82017-02-07 14:33:49 +0000995static void GenUnsafePut(HInvoke* invoke,
Andreas Gampe878d58c2015-01-15 23:24:00 -0800996 Primitive::Type type,
997 bool is_volatile,
998 bool is_ordered,
999 CodeGeneratorARM64* codegen) {
Artem Serov914d7a82017-02-07 14:33:49 +00001000 LocationSummary* locations = invoke->GetLocations();
Alexandre Rames087930f2016-08-02 13:45:28 +01001001 MacroAssembler* masm = codegen->GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -08001002
1003 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer.
1004 Register offset = XRegisterFrom(locations->InAt(2)); // Long offset.
1005 Register value = RegisterFrom(locations->InAt(3), type);
Roland Levillain4d027112015-07-01 15:41:14 +01001006 Register source = value;
Andreas Gampe878d58c2015-01-15 23:24:00 -08001007 MemOperand mem_op(base.X(), offset);
1008
Roland Levillain4d027112015-07-01 15:41:14 +01001009 {
1010 // We use a block to end the scratch scope before the write barrier, thus
1011 // freeing the temporary registers so they can be used in `MarkGCCard`.
1012 UseScratchRegisterScope temps(masm);
1013
1014 if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1015 DCHECK(value.IsW());
1016 Register temp = temps.AcquireW();
1017 __ Mov(temp.W(), value.W());
1018 codegen->GetAssembler()->PoisonHeapReference(temp.W());
1019 source = temp;
Andreas Gampe878d58c2015-01-15 23:24:00 -08001020 }
Roland Levillain4d027112015-07-01 15:41:14 +01001021
1022 if (is_volatile || is_ordered) {
Artem Serov914d7a82017-02-07 14:33:49 +00001023 codegen->StoreRelease(invoke, type, source, mem_op, /* needs_null_check */ false);
Roland Levillain4d027112015-07-01 15:41:14 +01001024 } else {
1025 codegen->Store(type, source, mem_op);
1026 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001027 }
1028
1029 if (type == Primitive::kPrimNot) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001030 bool value_can_be_null = true; // TODO: Worth finding out this information?
1031 codegen->MarkGCCard(base, value, value_can_be_null);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001032 }
1033}
1034
1035void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001036 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001037 Primitive::kPrimInt,
1038 /* is_volatile */ false,
1039 /* is_ordered */ false,
1040 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001041}
1042void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001043 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001044 Primitive::kPrimInt,
1045 /* is_volatile */ false,
1046 /* is_ordered */ true,
1047 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001048}
1049void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001050 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001051 Primitive::kPrimInt,
1052 /* is_volatile */ true,
1053 /* is_ordered */ false,
1054 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001055}
1056void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001057 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001058 Primitive::kPrimNot,
1059 /* is_volatile */ false,
1060 /* is_ordered */ false,
1061 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001062}
1063void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001064 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001065 Primitive::kPrimNot,
1066 /* is_volatile */ false,
1067 /* is_ordered */ true,
1068 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001069}
1070void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001071 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001072 Primitive::kPrimNot,
1073 /* is_volatile */ true,
1074 /* is_ordered */ false,
1075 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001076}
1077void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001078 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001079 Primitive::kPrimLong,
1080 /* is_volatile */ false,
1081 /* is_ordered */ false,
1082 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001083}
1084void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001085 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001086 Primitive::kPrimLong,
1087 /* is_volatile */ false,
1088 /* is_ordered */ true,
1089 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001090}
1091void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001092 GenUnsafePut(invoke,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001093 Primitive::kPrimLong,
1094 /* is_volatile */ true,
1095 /* is_ordered */ false,
1096 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001097}
1098
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001099static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena,
1100 HInvoke* invoke,
1101 Primitive::Type type) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001102 bool can_call = kEmitCompilerReadBarrier &&
1103 kUseBakerReadBarrier &&
1104 (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001105 LocationSummary* locations = new (arena) LocationSummary(invoke,
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001106 (can_call
1107 ? LocationSummary::kCallOnSlowPath
1108 : LocationSummary::kNoCall),
Andreas Gampe878d58c2015-01-15 23:24:00 -08001109 kIntrinsified);
1110 locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
1111 locations->SetInAt(1, Location::RequiresRegister());
1112 locations->SetInAt(2, Location::RequiresRegister());
1113 locations->SetInAt(3, Location::RequiresRegister());
1114 locations->SetInAt(4, Location::RequiresRegister());
1115
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001116 // If heap poisoning is enabled, we don't want the unpoisoning
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001117 // operations to potentially clobber the output. Likewise when
1118 // emitting a (Baker) read barrier, which may call.
1119 Location::OutputOverlap overlaps =
1120 ((kPoisonHeapReferences && type == Primitive::kPrimNot) || can_call)
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001121 ? Location::kOutputOverlap
1122 : Location::kNoOutputOverlap;
1123 locations->SetOut(Location::RequiresRegister(), overlaps);
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001124 if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
1125 // Temporary register for (Baker) read barrier.
1126 locations->AddTemp(Location::RequiresRegister());
1127 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001128}
1129
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001130static void GenCas(HInvoke* invoke, Primitive::Type type, CodeGeneratorARM64* codegen) {
Alexandre Rames087930f2016-08-02 13:45:28 +01001131 MacroAssembler* masm = codegen->GetVIXLAssembler();
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001132 LocationSummary* locations = invoke->GetLocations();
Andreas Gampe878d58c2015-01-15 23:24:00 -08001133
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001134 Location out_loc = locations->Out();
1135 Register out = WRegisterFrom(out_loc); // Boolean result.
Andreas Gampe878d58c2015-01-15 23:24:00 -08001136
1137 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer.
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001138 Location offset_loc = locations->InAt(2);
1139 Register offset = XRegisterFrom(offset_loc); // Long offset.
Andreas Gampe878d58c2015-01-15 23:24:00 -08001140 Register expected = RegisterFrom(locations->InAt(3), type); // Expected.
1141 Register value = RegisterFrom(locations->InAt(4), type); // Value.
1142
1143 // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
1144 if (type == Primitive::kPrimNot) {
1145 // Mark card for object assuming new value is stored.
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001146 bool value_can_be_null = true; // TODO: Worth finding out this information?
1147 codegen->MarkGCCard(base, value, value_can_be_null);
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001148
1149 // The only read barrier implementation supporting the
1150 // UnsafeCASObject intrinsic is the Baker-style read barriers.
1151 DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier);
1152
1153 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
1154 Register temp = WRegisterFrom(locations->GetTemp(0));
1155 // Need to make sure the reference stored in the field is a to-space
1156 // one before attempting the CAS or the CAS could fail incorrectly.
Roland Levillainff487002017-03-07 16:50:01 +00001157 codegen->UpdateReferenceFieldWithBakerReadBarrier(
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001158 invoke,
1159 out_loc, // Unused, used only as a "temporary" within the read barrier.
1160 base,
Roland Levillainff487002017-03-07 16:50:01 +00001161 /* field_offset */ offset_loc,
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001162 temp,
1163 /* needs_null_check */ false,
Roland Levillainff487002017-03-07 16:50:01 +00001164 /* use_load_acquire */ false);
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001165 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001166 }
1167
1168 UseScratchRegisterScope temps(masm);
1169 Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory.
1170 Register tmp_value = temps.AcquireSameSizeAs(value); // Value in memory.
1171
1172 Register tmp_32 = tmp_value.W();
1173
1174 __ Add(tmp_ptr, base.X(), Operand(offset));
1175
Roland Levillain4d027112015-07-01 15:41:14 +01001176 if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1177 codegen->GetAssembler()->PoisonHeapReference(expected);
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001178 if (value.Is(expected)) {
1179 // Do not poison `value`, as it is the same register as
1180 // `expected`, which has just been poisoned.
1181 } else {
1182 codegen->GetAssembler()->PoisonHeapReference(value);
1183 }
Roland Levillain4d027112015-07-01 15:41:14 +01001184 }
1185
Andreas Gampe878d58c2015-01-15 23:24:00 -08001186 // do {
1187 // tmp_value = [tmp_ptr] - expected;
1188 // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1189 // result = tmp_value != 0;
1190
Scott Wakeling97c72b72016-06-24 16:19:36 +01001191 vixl::aarch64::Label loop_head, exit_loop;
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001192 __ Bind(&loop_head);
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001193 __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1194 __ Cmp(tmp_value, expected);
1195 __ B(&exit_loop, ne);
1196 __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1197 __ Cbnz(tmp_32, &loop_head);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001198 __ Bind(&exit_loop);
1199 __ Cset(out, eq);
Roland Levillain4d027112015-07-01 15:41:14 +01001200
1201 if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
Roland Levillain4d027112015-07-01 15:41:14 +01001202 codegen->GetAssembler()->UnpoisonHeapReference(expected);
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001203 if (value.Is(expected)) {
1204 // Do not unpoison `value`, as it is the same register as
1205 // `expected`, which has just been unpoisoned.
1206 } else {
1207 codegen->GetAssembler()->UnpoisonHeapReference(value);
1208 }
Roland Levillain4d027112015-07-01 15:41:14 +01001209 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001210}
1211
1212void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001213 CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimInt);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001214}
1215void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001216 CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimLong);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001217}
1218void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001219 // The only read barrier implementation supporting the
1220 // UnsafeCASObject intrinsic is the Baker-style read barriers.
1221 if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) {
Roland Levillain985ff702015-10-23 13:25:35 +01001222 return;
1223 }
1224
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001225 CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimNot);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001226}
1227
1228void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001229 GenCas(invoke, Primitive::kPrimInt, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001230}
1231void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001232 GenCas(invoke, Primitive::kPrimLong, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001233}
1234void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001235 // The only read barrier implementation supporting the
1236 // UnsafeCASObject intrinsic is the Baker-style read barriers.
1237 DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier);
Roland Levillain3d312422016-06-23 13:53:42 +01001238
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001239 GenCas(invoke, Primitive::kPrimNot, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001240}
1241
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001242void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001243 LocationSummary* locations = new (arena_) LocationSummary(invoke,
Scott Wakeling1f36f412016-04-21 11:13:45 +01001244 invoke->InputAt(1)->CanBeNull()
1245 ? LocationSummary::kCallOnSlowPath
1246 : LocationSummary::kNoCall,
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001247 kIntrinsified);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001248 locations->SetInAt(0, Location::RequiresRegister());
1249 locations->SetInAt(1, Location::RequiresRegister());
1250 locations->AddTemp(Location::RequiresRegister());
1251 locations->AddTemp(Location::RequiresRegister());
1252 locations->AddTemp(Location::RequiresRegister());
jessicahandojo05765752016-09-09 19:01:32 -07001253 // Need temporary registers for String compression's feature.
1254 if (mirror::kUseStringCompression) {
1255 locations->AddTemp(Location::RequiresRegister());
jessicahandojo05765752016-09-09 19:01:32 -07001256 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001257 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001258}
1259
1260void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001261 MacroAssembler* masm = GetVIXLAssembler();
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001262 LocationSummary* locations = invoke->GetLocations();
1263
Alexandre Rames2ea91532016-08-11 17:04:14 +01001264 Register str = InputRegisterAt(invoke, 0);
1265 Register arg = InputRegisterAt(invoke, 1);
1266 DCHECK(str.IsW());
1267 DCHECK(arg.IsW());
Scott Wakeling1f36f412016-04-21 11:13:45 +01001268 Register out = OutputRegister(invoke);
1269
1270 Register temp0 = WRegisterFrom(locations->GetTemp(0));
1271 Register temp1 = WRegisterFrom(locations->GetTemp(1));
1272 Register temp2 = WRegisterFrom(locations->GetTemp(2));
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001273 Register temp3;
jessicahandojo05765752016-09-09 19:01:32 -07001274 if (mirror::kUseStringCompression) {
1275 temp3 = WRegisterFrom(locations->GetTemp(3));
jessicahandojo05765752016-09-09 19:01:32 -07001276 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001277
Scott Wakeling97c72b72016-06-24 16:19:36 +01001278 vixl::aarch64::Label loop;
1279 vixl::aarch64::Label find_char_diff;
1280 vixl::aarch64::Label end;
jessicahandojo05765752016-09-09 19:01:32 -07001281 vixl::aarch64::Label different_compression;
Scott Wakeling1f36f412016-04-21 11:13:45 +01001282
1283 // Get offsets of count and value fields within a string object.
1284 const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1285 const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1286
Nicolas Geoffray512e04d2015-03-27 17:21:24 +00001287 // Note that the null check must have been done earlier.
Calin Juravle641547a2015-04-21 22:08:51 +01001288 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001289
Scott Wakeling1f36f412016-04-21 11:13:45 +01001290 // Take slow path and throw if input can be and is null.
1291 SlowPathCodeARM64* slow_path = nullptr;
1292 const bool can_slow_path = invoke->InputAt(1)->CanBeNull();
1293 if (can_slow_path) {
1294 slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1295 codegen_->AddSlowPath(slow_path);
1296 __ Cbz(arg, slow_path->GetEntryLabel());
1297 }
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001298
Scott Wakeling1f36f412016-04-21 11:13:45 +01001299 // Reference equality check, return 0 if same reference.
1300 __ Subs(out, str, arg);
1301 __ B(&end, eq);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001302
jessicahandojo05765752016-09-09 19:01:32 -07001303 if (mirror::kUseStringCompression) {
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001304 // Load `count` fields of this and argument strings.
jessicahandojo05765752016-09-09 19:01:32 -07001305 __ Ldr(temp3, HeapOperand(str, count_offset));
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001306 __ Ldr(temp2, HeapOperand(arg, count_offset));
jessicahandojo05765752016-09-09 19:01:32 -07001307 // Clean out compression flag from lengths.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001308 __ Lsr(temp0, temp3, 1u);
1309 __ Lsr(temp1, temp2, 1u);
jessicahandojo05765752016-09-09 19:01:32 -07001310 } else {
1311 // Load lengths of this and argument strings.
1312 __ Ldr(temp0, HeapOperand(str, count_offset));
1313 __ Ldr(temp1, HeapOperand(arg, count_offset));
1314 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001315 // out = length diff.
1316 __ Subs(out, temp0, temp1);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001317 // temp0 = min(len(str), len(arg)).
1318 __ Csel(temp0, temp1, temp0, ge);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001319 // Shorter string is empty?
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001320 __ Cbz(temp0, &end);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001321
jessicahandojo05765752016-09-09 19:01:32 -07001322 if (mirror::kUseStringCompression) {
1323 // Check if both strings using same compression style to use this comparison loop.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001324 __ Eor(temp2, temp2, Operand(temp3));
1325 // Interleave with compression flag extraction which is needed for both paths
1326 // and also set flags which is needed only for the different compressions path.
1327 __ Ands(temp3.W(), temp3.W(), Operand(1));
1328 __ Tbnz(temp2, 0, &different_compression); // Does not use flags.
jessicahandojo05765752016-09-09 19:01:32 -07001329 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001330 // Store offset of string value in preparation for comparison loop.
1331 __ Mov(temp1, value_offset);
jessicahandojo05765752016-09-09 19:01:32 -07001332 if (mirror::kUseStringCompression) {
1333 // For string compression, calculate the number of bytes to compare (not chars).
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001334 // This could in theory exceed INT32_MAX, so treat temp0 as unsigned.
1335 __ Lsl(temp0, temp0, temp3);
jessicahandojo05765752016-09-09 19:01:32 -07001336 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001337
1338 UseScratchRegisterScope scratch_scope(masm);
1339 Register temp4 = scratch_scope.AcquireX();
1340
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001341 // Assertions that must hold in order to compare strings 8 bytes at a time.
Scott Wakeling1f36f412016-04-21 11:13:45 +01001342 DCHECK_ALIGNED(value_offset, 8);
1343 static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1344
1345 const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar);
1346 DCHECK_EQ(char_size, 2u);
1347
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001348 // Promote temp2 to an X reg, ready for LDR.
1349 temp2 = temp2.X();
Scott Wakeling1f36f412016-04-21 11:13:45 +01001350
1351 // Loop to compare 4x16-bit characters at a time (ok because of string data alignment).
1352 __ Bind(&loop);
Alexandre Rames2ea91532016-08-11 17:04:14 +01001353 __ Ldr(temp4, MemOperand(str.X(), temp1.X()));
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001354 __ Ldr(temp2, MemOperand(arg.X(), temp1.X()));
1355 __ Cmp(temp4, temp2);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001356 __ B(ne, &find_char_diff);
1357 __ Add(temp1, temp1, char_size * 4);
jessicahandojo05765752016-09-09 19:01:32 -07001358 // With string compression, we have compared 8 bytes, otherwise 4 chars.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001359 __ Subs(temp0, temp0, (mirror::kUseStringCompression) ? 8 : 4);
1360 __ B(&loop, hi);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001361 __ B(&end);
1362
1363 // Promote temp1 to an X reg, ready for EOR.
1364 temp1 = temp1.X();
1365
jessicahandojo05765752016-09-09 19:01:32 -07001366 // Find the single character difference.
Scott Wakeling1f36f412016-04-21 11:13:45 +01001367 __ Bind(&find_char_diff);
1368 // Get the bit position of the first character that differs.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001369 __ Eor(temp1, temp2, temp4);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001370 __ Rbit(temp1, temp1);
1371 __ Clz(temp1, temp1);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001372
jessicahandojo05765752016-09-09 19:01:32 -07001373 // If the number of chars remaining <= the index where the difference occurs (0-3), then
Scott Wakeling1f36f412016-04-21 11:13:45 +01001374 // the difference occurs outside the remaining string data, so just return length diff (out).
jessicahandojo05765752016-09-09 19:01:32 -07001375 // Unlike ARM, we're doing the comparison in one go here, without the subtraction at the
1376 // find_char_diff_2nd_cmp path, so it doesn't matter whether the comparison is signed or
1377 // unsigned when string compression is disabled.
1378 // When it's enabled, the comparison must be unsigned.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001379 __ Cmp(temp0, Operand(temp1.W(), LSR, (mirror::kUseStringCompression) ? 3 : 4));
jessicahandojo05765752016-09-09 19:01:32 -07001380 __ B(ls, &end);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001381
Scott Wakeling1f36f412016-04-21 11:13:45 +01001382 // Extract the characters and calculate the difference.
jessicahandojo05765752016-09-09 19:01:32 -07001383 if (mirror:: kUseStringCompression) {
jessicahandojo05765752016-09-09 19:01:32 -07001384 __ Bic(temp1, temp1, 0x7);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001385 __ Bic(temp1, temp1, Operand(temp3.X(), LSL, 3u));
1386 } else {
1387 __ Bic(temp1, temp1, 0xf);
jessicahandojo05765752016-09-09 19:01:32 -07001388 }
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001389 __ Lsr(temp2, temp2, temp1);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001390 __ Lsr(temp4, temp4, temp1);
jessicahandojo05765752016-09-09 19:01:32 -07001391 if (mirror::kUseStringCompression) {
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001392 // Prioritize the case of compressed strings and calculate such result first.
1393 __ Uxtb(temp1, temp4);
1394 __ Sub(out, temp1.W(), Operand(temp2.W(), UXTB));
1395 __ Tbz(temp3, 0u, &end); // If actually compressed, we're done.
jessicahandojo05765752016-09-09 19:01:32 -07001396 }
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001397 __ Uxth(temp4, temp4);
1398 __ Sub(out, temp4.W(), Operand(temp2.W(), UXTH));
jessicahandojo05765752016-09-09 19:01:32 -07001399
1400 if (mirror::kUseStringCompression) {
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001401 __ B(&end);
1402 __ Bind(&different_compression);
1403
1404 // Comparison for different compression style.
jessicahandojo05765752016-09-09 19:01:32 -07001405 const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte);
1406 DCHECK_EQ(c_char_size, 1u);
jessicahandojo05765752016-09-09 19:01:32 -07001407 temp1 = temp1.W();
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001408 temp2 = temp2.W();
1409 temp4 = temp4.W();
jessicahandojo05765752016-09-09 19:01:32 -07001410
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001411 // `temp1` will hold the compressed data pointer, `temp2` the uncompressed data pointer.
1412 // Note that flags have been set by the `str` compression flag extraction to `temp3`
1413 // before branching to the `different_compression` label.
1414 __ Csel(temp1, str, arg, eq); // Pointer to the compressed string.
1415 __ Csel(temp2, str, arg, ne); // Pointer to the uncompressed string.
jessicahandojo05765752016-09-09 19:01:32 -07001416
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001417 // We want to free up the temp3, currently holding `str` compression flag, for comparison.
1418 // So, we move it to the bottom bit of the iteration count `temp0` which we then need to treat
1419 // as unsigned. Start by freeing the bit with a LSL and continue further down by a SUB which
1420 // will allow `subs temp0, #2; bhi different_compression_loop` to serve as the loop condition.
1421 __ Lsl(temp0, temp0, 1u);
1422
1423 // Adjust temp1 and temp2 from string pointers to data pointers.
1424 __ Add(temp1, temp1, Operand(value_offset));
1425 __ Add(temp2, temp2, Operand(value_offset));
1426
1427 // Complete the move of the compression flag.
1428 __ Sub(temp0, temp0, Operand(temp3));
1429
1430 vixl::aarch64::Label different_compression_loop;
1431 vixl::aarch64::Label different_compression_diff;
1432
1433 __ Bind(&different_compression_loop);
1434 __ Ldrb(temp4, MemOperand(temp1.X(), c_char_size, PostIndex));
1435 __ Ldrh(temp3, MemOperand(temp2.X(), char_size, PostIndex));
1436 __ Subs(temp4, temp4, Operand(temp3));
1437 __ B(&different_compression_diff, ne);
1438 __ Subs(temp0, temp0, 2);
1439 __ B(&different_compression_loop, hi);
jessicahandojo05765752016-09-09 19:01:32 -07001440 __ B(&end);
1441
1442 // Calculate the difference.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001443 __ Bind(&different_compression_diff);
1444 __ Tst(temp0, Operand(1));
1445 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u,
1446 "Expecting 0=compressed, 1=uncompressed");
1447 __ Cneg(out, temp4, ne);
jessicahandojo05765752016-09-09 19:01:32 -07001448 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001449
1450 __ Bind(&end);
1451
1452 if (can_slow_path) {
1453 __ Bind(slow_path->GetExitLabel());
1454 }
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001455}
1456
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001457// The cut off for unrolling the loop in String.equals() intrinsic for const strings.
1458// The normal loop plus the pre-header is 9 instructions without string compression and 12
1459// instructions with string compression. We can compare up to 8 bytes in 4 instructions
1460// (LDR+LDR+CMP+BNE) and up to 16 bytes in 5 instructions (LDP+LDP+CMP+CCMP+BNE). Allow up
1461// to 10 instructions for the unrolled loop.
1462constexpr size_t kShortConstStringEqualsCutoffInBytes = 32;
1463
1464static const char* GetConstString(HInstruction* candidate, uint32_t* utf16_length) {
1465 if (candidate->IsLoadString()) {
1466 HLoadString* load_string = candidate->AsLoadString();
1467 const DexFile& dex_file = load_string->GetDexFile();
1468 return dex_file.StringDataAndUtf16LengthByIdx(load_string->GetStringIndex(), utf16_length);
1469 }
1470 return nullptr;
1471}
1472
Agi Csakiea34b402015-08-13 17:51:19 -07001473void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
1474 LocationSummary* locations = new (arena_) LocationSummary(invoke,
1475 LocationSummary::kNoCall,
1476 kIntrinsified);
1477 locations->SetInAt(0, Location::RequiresRegister());
1478 locations->SetInAt(1, Location::RequiresRegister());
Agi Csakiea34b402015-08-13 17:51:19 -07001479
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001480 // For the generic implementation and for long const strings we need a temporary.
1481 // We do not need it for short const strings, up to 8 bytes, see code generation below.
1482 uint32_t const_string_length = 0u;
1483 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length);
1484 if (const_string == nullptr) {
1485 const_string = GetConstString(invoke->InputAt(1), &const_string_length);
1486 }
1487 bool is_compressed =
1488 mirror::kUseStringCompression &&
1489 const_string != nullptr &&
1490 mirror::String::DexFileStringAllASCII(const_string, const_string_length);
1491 if (const_string == nullptr || const_string_length > (is_compressed ? 8u : 4u)) {
1492 locations->AddTemp(Location::RequiresRegister());
1493 }
1494
1495 // TODO: If the String.equals() is used only for an immediately following HIf, we can
1496 // mark it as emitted-at-use-site and emit branches directly to the appropriate blocks.
1497 // Then we shall need an extra temporary register instead of the output register.
Agi Csakiea34b402015-08-13 17:51:19 -07001498 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1499}
1500
1501void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001502 MacroAssembler* masm = GetVIXLAssembler();
Agi Csakiea34b402015-08-13 17:51:19 -07001503 LocationSummary* locations = invoke->GetLocations();
1504
1505 Register str = WRegisterFrom(locations->InAt(0));
1506 Register arg = WRegisterFrom(locations->InAt(1));
1507 Register out = XRegisterFrom(locations->Out());
1508
1509 UseScratchRegisterScope scratch_scope(masm);
1510 Register temp = scratch_scope.AcquireW();
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001511 Register temp1 = scratch_scope.AcquireW();
Agi Csakiea34b402015-08-13 17:51:19 -07001512
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001513 vixl::aarch64::Label loop;
Scott Wakeling97c72b72016-06-24 16:19:36 +01001514 vixl::aarch64::Label end;
1515 vixl::aarch64::Label return_true;
1516 vixl::aarch64::Label return_false;
Agi Csakiea34b402015-08-13 17:51:19 -07001517
1518 // Get offsets of count, value, and class fields within a string object.
1519 const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1520 const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1521 const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1522
1523 // Note that the null check must have been done earlier.
1524 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1525
Vladimir Marko53b52002016-05-24 19:30:45 +01001526 StringEqualsOptimizations optimizations(invoke);
1527 if (!optimizations.GetArgumentNotNull()) {
1528 // Check if input is null, return false if it is.
1529 __ Cbz(arg, &return_false);
1530 }
Agi Csakiea34b402015-08-13 17:51:19 -07001531
1532 // Reference equality check, return true if same reference.
1533 __ Cmp(str, arg);
1534 __ B(&return_true, eq);
1535
Vladimir Marko53b52002016-05-24 19:30:45 +01001536 if (!optimizations.GetArgumentIsString()) {
1537 // Instanceof check for the argument by comparing class fields.
1538 // All string objects must have the same type since String cannot be subclassed.
1539 // Receiver must be a string object, so its class field is equal to all strings' class fields.
1540 // If the argument is a string object, its class field must be equal to receiver's class field.
1541 __ Ldr(temp, MemOperand(str.X(), class_offset));
1542 __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1543 __ Cmp(temp, temp1);
1544 __ B(&return_false, ne);
1545 }
Agi Csakiea34b402015-08-13 17:51:19 -07001546
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001547 // Check if one of the inputs is a const string. Do not special-case both strings
1548 // being const, such cases should be handled by constant folding if needed.
1549 uint32_t const_string_length = 0u;
1550 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length);
1551 if (const_string == nullptr) {
1552 const_string = GetConstString(invoke->InputAt(1), &const_string_length);
1553 if (const_string != nullptr) {
1554 std::swap(str, arg); // Make sure the const string is in `str`.
1555 }
1556 }
1557 bool is_compressed =
1558 mirror::kUseStringCompression &&
1559 const_string != nullptr &&
1560 mirror::String::DexFileStringAllASCII(const_string, const_string_length);
1561
1562 if (const_string != nullptr) {
1563 // Load `count` field of the argument string and check if it matches the const string.
1564 // Also compares the compression style, if differs return false.
1565 __ Ldr(temp, MemOperand(arg.X(), count_offset));
Vladimir Marko26ec3ca2017-03-14 13:37:14 +00001566 // Temporarily release temp1 as we may not be able to embed the flagged count in CMP immediate.
1567 scratch_scope.Release(temp1);
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001568 __ Cmp(temp, Operand(mirror::String::GetFlaggedCount(const_string_length, is_compressed)));
Vladimir Marko26ec3ca2017-03-14 13:37:14 +00001569 temp1 = scratch_scope.AcquireW();
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001570 __ B(&return_false, ne);
1571 } else {
1572 // Load `count` fields of this and argument strings.
1573 __ Ldr(temp, MemOperand(str.X(), count_offset));
1574 __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1575 // Check if `count` fields are equal, return false if they're not.
1576 // Also compares the compression style, if differs return false.
1577 __ Cmp(temp, temp1);
1578 __ B(&return_false, ne);
1579 }
Agi Csakiea34b402015-08-13 17:51:19 -07001580
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001581 // Assertions that must hold in order to compare strings 8 bytes at a time.
Agi Csakiea34b402015-08-13 17:51:19 -07001582 DCHECK_ALIGNED(value_offset, 8);
1583 static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1584
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001585 if (const_string != nullptr &&
1586 const_string_length < (is_compressed ? kShortConstStringEqualsCutoffInBytes
1587 : kShortConstStringEqualsCutoffInBytes / 2u)) {
1588 // Load and compare the contents. Though we know the contents of the short const string
1589 // at compile time, materializing constants may be more code than loading from memory.
1590 int32_t offset = value_offset;
1591 size_t remaining_bytes =
1592 RoundUp(is_compressed ? const_string_length : const_string_length * 2u, 8u);
1593 temp = temp.X();
1594 temp1 = temp1.X();
1595 while (remaining_bytes > 8u) {
1596 Register temp2 = XRegisterFrom(locations->GetTemp(0));
1597 __ Ldp(temp, temp1, MemOperand(str.X(), offset));
1598 __ Ldp(temp2, out, MemOperand(arg.X(), offset));
1599 __ Cmp(temp, temp2);
1600 __ Ccmp(temp1, out, NoFlag, eq);
1601 __ B(&return_false, ne);
1602 offset += 2u * sizeof(uint64_t);
1603 remaining_bytes -= 2u * sizeof(uint64_t);
1604 }
1605 if (remaining_bytes != 0u) {
1606 __ Ldr(temp, MemOperand(str.X(), offset));
1607 __ Ldr(temp1, MemOperand(arg.X(), offset));
1608 __ Cmp(temp, temp1);
1609 __ B(&return_false, ne);
1610 }
1611 } else {
1612 // Return true if both strings are empty. Even with string compression `count == 0` means empty.
1613 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u,
1614 "Expecting 0=compressed, 1=uncompressed");
1615 __ Cbz(temp, &return_true);
1616
1617 if (mirror::kUseStringCompression) {
1618 // For string compression, calculate the number of bytes to compare (not chars).
1619 // This could in theory exceed INT32_MAX, so treat temp as unsigned.
1620 __ And(temp1, temp, Operand(1)); // Extract compression flag.
1621 __ Lsr(temp, temp, 1u); // Extract length.
1622 __ Lsl(temp, temp, temp1); // Calculate number of bytes to compare.
1623 }
1624
1625 // Store offset of string value in preparation for comparison loop
1626 __ Mov(temp1, value_offset);
1627
1628 temp1 = temp1.X();
1629 Register temp2 = XRegisterFrom(locations->GetTemp(0));
1630 // Loop to compare strings 8 bytes at a time starting at the front of the string.
1631 // Ok to do this because strings are zero-padded to kObjectAlignment.
1632 __ Bind(&loop);
1633 __ Ldr(out, MemOperand(str.X(), temp1));
1634 __ Ldr(temp2, MemOperand(arg.X(), temp1));
1635 __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1636 __ Cmp(out, temp2);
1637 __ B(&return_false, ne);
1638 // With string compression, we have compared 8 bytes, otherwise 4 chars.
1639 __ Sub(temp, temp, Operand(mirror::kUseStringCompression ? 8 : 4), SetFlags);
1640 __ B(&loop, hi);
jessicahandojo05765752016-09-09 19:01:32 -07001641 }
1642
Agi Csakiea34b402015-08-13 17:51:19 -07001643 // Return true and exit the function.
1644 // If loop does not result in returning false, we return true.
1645 __ Bind(&return_true);
1646 __ Mov(out, 1);
1647 __ B(&end);
1648
1649 // Return false and exit the function.
1650 __ Bind(&return_false);
1651 __ Mov(out, 0);
1652 __ Bind(&end);
1653}
1654
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001655static void GenerateVisitStringIndexOf(HInvoke* invoke,
Scott Wakeling97c72b72016-06-24 16:19:36 +01001656 MacroAssembler* masm,
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001657 CodeGeneratorARM64* codegen,
1658 ArenaAllocator* allocator,
1659 bool start_at_zero) {
1660 LocationSummary* locations = invoke->GetLocations();
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001661
1662 // Note that the null check must have been done earlier.
1663 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1664
1665 // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001666 // or directly dispatch for a large constant, or omit slow-path for a small constant or a char.
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001667 SlowPathCodeARM64* slow_path = nullptr;
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001668 HInstruction* code_point = invoke->InputAt(1);
1669 if (code_point->IsIntConstant()) {
Vladimir Markoda051082016-05-17 16:10:20 +01001670 if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > 0xFFFFU) {
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001671 // Always needs the slow-path. We could directly dispatch to it, but this case should be
1672 // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1673 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1674 codegen->AddSlowPath(slow_path);
1675 __ B(slow_path->GetEntryLabel());
1676 __ Bind(slow_path->GetExitLabel());
1677 return;
1678 }
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001679 } else if (code_point->GetType() != Primitive::kPrimChar) {
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001680 Register char_reg = WRegisterFrom(locations->InAt(1));
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001681 __ Tst(char_reg, 0xFFFF0000);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001682 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1683 codegen->AddSlowPath(slow_path);
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001684 __ B(ne, slow_path->GetEntryLabel());
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001685 }
1686
1687 if (start_at_zero) {
1688 // Start-index = 0.
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001689 Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001690 __ Mov(tmp_reg, 0);
1691 }
1692
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001693 codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path);
Roland Levillain42ad2882016-02-29 18:26:54 +00001694 CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>();
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001695
1696 if (slow_path != nullptr) {
1697 __ Bind(slow_path->GetExitLabel());
1698 }
1699}
1700
1701void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1702 LocationSummary* locations = new (arena_) LocationSummary(invoke,
Serban Constantinescu806f0122016-03-09 11:10:16 +00001703 LocationSummary::kCallOnMainAndSlowPath,
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001704 kIntrinsified);
1705 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1706 // best to align the inputs accordingly.
1707 InvokeRuntimeCallingConvention calling_convention;
1708 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1709 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1710 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1711
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001712 // Need to send start_index=0.
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001713 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1714}
1715
1716void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001717 GenerateVisitStringIndexOf(
1718 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001719}
1720
1721void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1722 LocationSummary* locations = new (arena_) LocationSummary(invoke,
Serban Constantinescu806f0122016-03-09 11:10:16 +00001723 LocationSummary::kCallOnMainAndSlowPath,
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001724 kIntrinsified);
1725 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1726 // best to align the inputs accordingly.
1727 InvokeRuntimeCallingConvention calling_convention;
1728 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1729 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1730 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1731 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001732}
1733
1734void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001735 GenerateVisitStringIndexOf(
1736 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001737}
1738
Jeff Hao848f70a2014-01-15 13:49:50 -08001739void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1740 LocationSummary* locations = new (arena_) LocationSummary(invoke,
Serban Constantinescu806f0122016-03-09 11:10:16 +00001741 LocationSummary::kCallOnMainAndSlowPath,
Jeff Hao848f70a2014-01-15 13:49:50 -08001742 kIntrinsified);
1743 InvokeRuntimeCallingConvention calling_convention;
1744 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1745 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1746 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1747 locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1748 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1749}
1750
1751void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001752 MacroAssembler* masm = GetVIXLAssembler();
Jeff Hao848f70a2014-01-15 13:49:50 -08001753 LocationSummary* locations = invoke->GetLocations();
1754
1755 Register byte_array = WRegisterFrom(locations->InAt(0));
1756 __ Cmp(byte_array, 0);
1757 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1758 codegen_->AddSlowPath(slow_path);
1759 __ B(eq, slow_path->GetEntryLabel());
1760
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001761 codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc(), slow_path);
Roland Levillainf969a202016-03-09 16:14:00 +00001762 CheckEntrypointTypes<kQuickAllocStringFromBytes, void*, void*, int32_t, int32_t, int32_t>();
Jeff Hao848f70a2014-01-15 13:49:50 -08001763 __ Bind(slow_path->GetExitLabel());
1764}
1765
1766void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1767 LocationSummary* locations = new (arena_) LocationSummary(invoke,
Serban Constantinescu54ff4822016-07-07 18:03:19 +01001768 LocationSummary::kCallOnMainOnly,
Jeff Hao848f70a2014-01-15 13:49:50 -08001769 kIntrinsified);
1770 InvokeRuntimeCallingConvention calling_convention;
1771 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1772 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1773 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1774 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1775}
1776
1777void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
Roland Levillaincc3839c2016-02-29 16:23:48 +00001778 // No need to emit code checking whether `locations->InAt(2)` is a null
1779 // pointer, as callers of the native method
1780 //
1781 // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data)
1782 //
1783 // all include a null check on `data` before calling that method.
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001784 codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc());
Roland Levillainf969a202016-03-09 16:14:00 +00001785 CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>();
Jeff Hao848f70a2014-01-15 13:49:50 -08001786}
1787
1788void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
Jeff Hao848f70a2014-01-15 13:49:50 -08001789 LocationSummary* locations = new (arena_) LocationSummary(invoke,
Serban Constantinescu806f0122016-03-09 11:10:16 +00001790 LocationSummary::kCallOnMainAndSlowPath,
Jeff Hao848f70a2014-01-15 13:49:50 -08001791 kIntrinsified);
1792 InvokeRuntimeCallingConvention calling_convention;
1793 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
Jeff Hao848f70a2014-01-15 13:49:50 -08001794 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1795}
1796
1797void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001798 MacroAssembler* masm = GetVIXLAssembler();
Jeff Hao848f70a2014-01-15 13:49:50 -08001799 LocationSummary* locations = invoke->GetLocations();
1800
1801 Register string_to_copy = WRegisterFrom(locations->InAt(0));
1802 __ Cmp(string_to_copy, 0);
1803 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1804 codegen_->AddSlowPath(slow_path);
1805 __ B(eq, slow_path->GetEntryLabel());
1806
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001807 codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc(), slow_path);
Roland Levillainf969a202016-03-09 16:14:00 +00001808 CheckEntrypointTypes<kQuickAllocStringFromString, void*, void*>();
Jeff Hao848f70a2014-01-15 13:49:50 -08001809 __ Bind(slow_path->GetExitLabel());
1810}
1811
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001812static void CreateFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) {
1813 DCHECK_EQ(invoke->GetNumberOfArguments(), 1U);
1814 DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1815 DCHECK(Primitive::IsFloatingPointType(invoke->GetType()));
1816
1817 LocationSummary* const locations = new (arena) LocationSummary(invoke,
Serban Constantinescu54ff4822016-07-07 18:03:19 +01001818 LocationSummary::kCallOnMainOnly,
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001819 kIntrinsified);
1820 InvokeRuntimeCallingConvention calling_convention;
1821
1822 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1823 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1824}
1825
1826static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) {
1827 DCHECK_EQ(invoke->GetNumberOfArguments(), 2U);
1828 DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1829 DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(1)->GetType()));
1830 DCHECK(Primitive::IsFloatingPointType(invoke->GetType()));
1831
1832 LocationSummary* const locations = new (arena) LocationSummary(invoke,
Serban Constantinescu54ff4822016-07-07 18:03:19 +01001833 LocationSummary::kCallOnMainOnly,
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001834 kIntrinsified);
1835 InvokeRuntimeCallingConvention calling_convention;
1836
1837 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1838 locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
1839 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1840}
1841
1842static void GenFPToFPCall(HInvoke* invoke,
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001843 CodeGeneratorARM64* codegen,
1844 QuickEntrypointEnum entry) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001845 codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc());
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001846}
1847
1848void IntrinsicLocationsBuilderARM64::VisitMathCos(HInvoke* invoke) {
1849 CreateFPToFPCallLocations(arena_, invoke);
1850}
1851
1852void IntrinsicCodeGeneratorARM64::VisitMathCos(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001853 GenFPToFPCall(invoke, codegen_, kQuickCos);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001854}
1855
1856void IntrinsicLocationsBuilderARM64::VisitMathSin(HInvoke* invoke) {
1857 CreateFPToFPCallLocations(arena_, invoke);
1858}
1859
1860void IntrinsicCodeGeneratorARM64::VisitMathSin(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001861 GenFPToFPCall(invoke, codegen_, kQuickSin);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001862}
1863
1864void IntrinsicLocationsBuilderARM64::VisitMathAcos(HInvoke* invoke) {
1865 CreateFPToFPCallLocations(arena_, invoke);
1866}
1867
1868void IntrinsicCodeGeneratorARM64::VisitMathAcos(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001869 GenFPToFPCall(invoke, codegen_, kQuickAcos);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001870}
1871
1872void IntrinsicLocationsBuilderARM64::VisitMathAsin(HInvoke* invoke) {
1873 CreateFPToFPCallLocations(arena_, invoke);
1874}
1875
1876void IntrinsicCodeGeneratorARM64::VisitMathAsin(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001877 GenFPToFPCall(invoke, codegen_, kQuickAsin);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001878}
1879
1880void IntrinsicLocationsBuilderARM64::VisitMathAtan(HInvoke* invoke) {
1881 CreateFPToFPCallLocations(arena_, invoke);
1882}
1883
1884void IntrinsicCodeGeneratorARM64::VisitMathAtan(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001885 GenFPToFPCall(invoke, codegen_, kQuickAtan);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001886}
1887
1888void IntrinsicLocationsBuilderARM64::VisitMathCbrt(HInvoke* invoke) {
1889 CreateFPToFPCallLocations(arena_, invoke);
1890}
1891
1892void IntrinsicCodeGeneratorARM64::VisitMathCbrt(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001893 GenFPToFPCall(invoke, codegen_, kQuickCbrt);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001894}
1895
1896void IntrinsicLocationsBuilderARM64::VisitMathCosh(HInvoke* invoke) {
1897 CreateFPToFPCallLocations(arena_, invoke);
1898}
1899
1900void IntrinsicCodeGeneratorARM64::VisitMathCosh(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001901 GenFPToFPCall(invoke, codegen_, kQuickCosh);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001902}
1903
1904void IntrinsicLocationsBuilderARM64::VisitMathExp(HInvoke* invoke) {
1905 CreateFPToFPCallLocations(arena_, invoke);
1906}
1907
1908void IntrinsicCodeGeneratorARM64::VisitMathExp(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001909 GenFPToFPCall(invoke, codegen_, kQuickExp);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001910}
1911
1912void IntrinsicLocationsBuilderARM64::VisitMathExpm1(HInvoke* invoke) {
1913 CreateFPToFPCallLocations(arena_, invoke);
1914}
1915
1916void IntrinsicCodeGeneratorARM64::VisitMathExpm1(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001917 GenFPToFPCall(invoke, codegen_, kQuickExpm1);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001918}
1919
1920void IntrinsicLocationsBuilderARM64::VisitMathLog(HInvoke* invoke) {
1921 CreateFPToFPCallLocations(arena_, invoke);
1922}
1923
1924void IntrinsicCodeGeneratorARM64::VisitMathLog(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001925 GenFPToFPCall(invoke, codegen_, kQuickLog);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001926}
1927
1928void IntrinsicLocationsBuilderARM64::VisitMathLog10(HInvoke* invoke) {
1929 CreateFPToFPCallLocations(arena_, invoke);
1930}
1931
1932void IntrinsicCodeGeneratorARM64::VisitMathLog10(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001933 GenFPToFPCall(invoke, codegen_, kQuickLog10);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001934}
1935
1936void IntrinsicLocationsBuilderARM64::VisitMathSinh(HInvoke* invoke) {
1937 CreateFPToFPCallLocations(arena_, invoke);
1938}
1939
1940void IntrinsicCodeGeneratorARM64::VisitMathSinh(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001941 GenFPToFPCall(invoke, codegen_, kQuickSinh);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001942}
1943
1944void IntrinsicLocationsBuilderARM64::VisitMathTan(HInvoke* invoke) {
1945 CreateFPToFPCallLocations(arena_, invoke);
1946}
1947
1948void IntrinsicCodeGeneratorARM64::VisitMathTan(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001949 GenFPToFPCall(invoke, codegen_, kQuickTan);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001950}
1951
1952void IntrinsicLocationsBuilderARM64::VisitMathTanh(HInvoke* invoke) {
1953 CreateFPToFPCallLocations(arena_, invoke);
1954}
1955
1956void IntrinsicCodeGeneratorARM64::VisitMathTanh(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001957 GenFPToFPCall(invoke, codegen_, kQuickTanh);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001958}
1959
1960void IntrinsicLocationsBuilderARM64::VisitMathAtan2(HInvoke* invoke) {
1961 CreateFPFPToFPCallLocations(arena_, invoke);
1962}
1963
1964void IntrinsicCodeGeneratorARM64::VisitMathAtan2(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001965 GenFPToFPCall(invoke, codegen_, kQuickAtan2);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001966}
1967
1968void IntrinsicLocationsBuilderARM64::VisitMathHypot(HInvoke* invoke) {
1969 CreateFPFPToFPCallLocations(arena_, invoke);
1970}
1971
1972void IntrinsicCodeGeneratorARM64::VisitMathHypot(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001973 GenFPToFPCall(invoke, codegen_, kQuickHypot);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001974}
1975
1976void IntrinsicLocationsBuilderARM64::VisitMathNextAfter(HInvoke* invoke) {
1977 CreateFPFPToFPCallLocations(arena_, invoke);
1978}
1979
1980void IntrinsicCodeGeneratorARM64::VisitMathNextAfter(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001981 GenFPToFPCall(invoke, codegen_, kQuickNextAfter);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001982}
1983
Tim Zhang25abd6c2016-01-19 23:39:24 +08001984void IntrinsicLocationsBuilderARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
1985 LocationSummary* locations = new (arena_) LocationSummary(invoke,
1986 LocationSummary::kNoCall,
1987 kIntrinsified);
1988 locations->SetInAt(0, Location::RequiresRegister());
1989 locations->SetInAt(1, Location::RequiresRegister());
1990 locations->SetInAt(2, Location::RequiresRegister());
1991 locations->SetInAt(3, Location::RequiresRegister());
1992 locations->SetInAt(4, Location::RequiresRegister());
1993
1994 locations->AddTemp(Location::RequiresRegister());
1995 locations->AddTemp(Location::RequiresRegister());
Scott Wakelingdf109d92016-04-22 11:35:56 +01001996 locations->AddTemp(Location::RequiresRegister());
Tim Zhang25abd6c2016-01-19 23:39:24 +08001997}
1998
1999void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01002000 MacroAssembler* masm = GetVIXLAssembler();
Tim Zhang25abd6c2016-01-19 23:39:24 +08002001 LocationSummary* locations = invoke->GetLocations();
2002
2003 // Check assumption that sizeof(Char) is 2 (used in scaling below).
2004 const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar);
2005 DCHECK_EQ(char_size, 2u);
2006
2007 // Location of data in char array buffer.
2008 const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value();
2009
2010 // Location of char array data in string.
2011 const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value();
2012
2013 // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin);
2014 // Since getChars() calls getCharsNoCheck() - we use registers rather than constants.
2015 Register srcObj = XRegisterFrom(locations->InAt(0));
2016 Register srcBegin = XRegisterFrom(locations->InAt(1));
2017 Register srcEnd = XRegisterFrom(locations->InAt(2));
2018 Register dstObj = XRegisterFrom(locations->InAt(3));
2019 Register dstBegin = XRegisterFrom(locations->InAt(4));
2020
2021 Register src_ptr = XRegisterFrom(locations->GetTemp(0));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002022 Register num_chr = XRegisterFrom(locations->GetTemp(1));
2023 Register tmp1 = XRegisterFrom(locations->GetTemp(2));
Tim Zhang25abd6c2016-01-19 23:39:24 +08002024
2025 UseScratchRegisterScope temps(masm);
2026 Register dst_ptr = temps.AcquireX();
Scott Wakelingdf109d92016-04-22 11:35:56 +01002027 Register tmp2 = temps.AcquireX();
Tim Zhang25abd6c2016-01-19 23:39:24 +08002028
jessicahandojo05765752016-09-09 19:01:32 -07002029 vixl::aarch64::Label done;
2030 vixl::aarch64::Label compressed_string_loop;
2031 __ Sub(num_chr, srcEnd, srcBegin);
2032 // Early out for valid zero-length retrievals.
2033 __ Cbz(num_chr, &done);
Tim Zhang25abd6c2016-01-19 23:39:24 +08002034
Scott Wakelingdf109d92016-04-22 11:35:56 +01002035 // dst address start to copy to.
Tim Zhang25abd6c2016-01-19 23:39:24 +08002036 __ Add(dst_ptr, dstObj, Operand(data_offset));
2037 __ Add(dst_ptr, dst_ptr, Operand(dstBegin, LSL, 1));
2038
jessicahandojo05765752016-09-09 19:01:32 -07002039 // src address to copy from.
2040 __ Add(src_ptr, srcObj, Operand(value_offset));
2041 vixl::aarch64::Label compressed_string_preloop;
2042 if (mirror::kUseStringCompression) {
2043 // Location of count in string.
2044 const uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
2045 // String's length.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01002046 __ Ldr(tmp2, MemOperand(srcObj, count_offset));
2047 __ Tbz(tmp2, 0, &compressed_string_preloop);
jessicahandojo05765752016-09-09 19:01:32 -07002048 }
2049 __ Add(src_ptr, src_ptr, Operand(srcBegin, LSL, 1));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002050
Tim Zhang25abd6c2016-01-19 23:39:24 +08002051 // Do the copy.
Scott Wakeling97c72b72016-06-24 16:19:36 +01002052 vixl::aarch64::Label loop;
Scott Wakeling97c72b72016-06-24 16:19:36 +01002053 vixl::aarch64::Label remainder;
Scott Wakelingdf109d92016-04-22 11:35:56 +01002054
Scott Wakelingdf109d92016-04-22 11:35:56 +01002055 // Save repairing the value of num_chr on the < 8 character path.
2056 __ Subs(tmp1, num_chr, 8);
2057 __ B(lt, &remainder);
2058
2059 // Keep the result of the earlier subs, we are going to fetch at least 8 characters.
2060 __ Mov(num_chr, tmp1);
2061
2062 // Main loop used for longer fetches loads and stores 8x16-bit characters at a time.
2063 // (Unaligned addresses are acceptable here and not worth inlining extra code to rectify.)
Tim Zhang25abd6c2016-01-19 23:39:24 +08002064 __ Bind(&loop);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002065 __ Ldp(tmp1, tmp2, MemOperand(src_ptr, char_size * 8, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002066 __ Subs(num_chr, num_chr, 8);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002067 __ Stp(tmp1, tmp2, MemOperand(dst_ptr, char_size * 8, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002068 __ B(ge, &loop);
2069
2070 __ Adds(num_chr, num_chr, 8);
2071 __ B(eq, &done);
2072
2073 // Main loop for < 8 character case and remainder handling. Loads and stores one
2074 // 16-bit Java character at a time.
2075 __ Bind(&remainder);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002076 __ Ldrh(tmp1, MemOperand(src_ptr, char_size, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002077 __ Subs(num_chr, num_chr, 1);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002078 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002079 __ B(gt, &remainder);
jessicahandojo05765752016-09-09 19:01:32 -07002080 __ B(&done);
2081
2082 if (mirror::kUseStringCompression) {
2083 const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte);
2084 DCHECK_EQ(c_char_size, 1u);
2085 __ Bind(&compressed_string_preloop);
2086 __ Add(src_ptr, src_ptr, Operand(srcBegin));
2087 // Copy loop for compressed src, copying 1 character (8-bit) to (16-bit) at a time.
2088 __ Bind(&compressed_string_loop);
2089 __ Ldrb(tmp1, MemOperand(src_ptr, c_char_size, PostIndex));
2090 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex));
2091 __ Subs(num_chr, num_chr, Operand(1));
2092 __ B(gt, &compressed_string_loop);
2093 }
Scott Wakelingdf109d92016-04-22 11:35:56 +01002094
Tim Zhang25abd6c2016-01-19 23:39:24 +08002095 __ Bind(&done);
2096}
2097
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002098// Mirrors ARRAYCOPY_SHORT_CHAR_ARRAY_THRESHOLD in libcore, so we can choose to use the native
2099// implementation there for longer copy lengths.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002100static constexpr int32_t kSystemArrayCopyCharThreshold = 32;
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002101
2102static void SetSystemArrayCopyLocationRequires(LocationSummary* locations,
2103 uint32_t at,
2104 HInstruction* input) {
2105 HIntConstant* const_input = input->AsIntConstant();
Scott Wakeling97c72b72016-06-24 16:19:36 +01002106 if (const_input != nullptr && !vixl::aarch64::Assembler::IsImmAddSub(const_input->GetValue())) {
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002107 locations->SetInAt(at, Location::RequiresRegister());
2108 } else {
2109 locations->SetInAt(at, Location::RegisterOrConstant(input));
2110 }
2111}
2112
2113void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopyChar(HInvoke* invoke) {
2114 // Check to see if we have known failures that will cause us to have to bail out
2115 // to the runtime, and just generate the runtime call directly.
2116 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
2117 HIntConstant* dst_pos = invoke->InputAt(3)->AsIntConstant();
2118
2119 // The positions must be non-negative.
2120 if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
2121 (dst_pos != nullptr && dst_pos->GetValue() < 0)) {
2122 // We will have to fail anyways.
2123 return;
2124 }
2125
2126 // The length must be >= 0 and not so long that we would (currently) prefer libcore's
2127 // native implementation.
2128 HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
2129 if (length != nullptr) {
2130 int32_t len = length->GetValue();
donghui.baic2ec9ad2016-03-10 14:02:55 +08002131 if (len < 0 || len > kSystemArrayCopyCharThreshold) {
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002132 // Just call as normal.
2133 return;
2134 }
2135 }
2136
2137 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
2138 LocationSummary* locations = new (allocator) LocationSummary(invoke,
2139 LocationSummary::kCallOnSlowPath,
2140 kIntrinsified);
2141 // arraycopy(char[] src, int src_pos, char[] dst, int dst_pos, int length).
2142 locations->SetInAt(0, Location::RequiresRegister());
2143 SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1));
2144 locations->SetInAt(2, Location::RequiresRegister());
2145 SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3));
2146 SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4));
2147
2148 locations->AddTemp(Location::RequiresRegister());
2149 locations->AddTemp(Location::RequiresRegister());
2150 locations->AddTemp(Location::RequiresRegister());
2151}
2152
Scott Wakeling97c72b72016-06-24 16:19:36 +01002153static void CheckSystemArrayCopyPosition(MacroAssembler* masm,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002154 const Location& pos,
2155 const Register& input,
2156 const Location& length,
2157 SlowPathCodeARM64* slow_path,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002158 const Register& temp,
2159 bool length_is_input_length = false) {
2160 const int32_t length_offset = mirror::Array::LengthOffset().Int32Value();
2161 if (pos.IsConstant()) {
2162 int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue();
2163 if (pos_const == 0) {
2164 if (!length_is_input_length) {
2165 // Check that length(input) >= length.
2166 __ Ldr(temp, MemOperand(input, length_offset));
2167 __ Cmp(temp, OperandFrom(length, Primitive::kPrimInt));
2168 __ B(slow_path->GetEntryLabel(), lt);
2169 }
2170 } else {
2171 // Check that length(input) >= pos.
Nicolas Geoffrayfea1abd2016-07-06 12:09:12 +01002172 __ Ldr(temp, MemOperand(input, length_offset));
2173 __ Subs(temp, temp, pos_const);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002174 __ B(slow_path->GetEntryLabel(), lt);
2175
2176 // Check that (length(input) - pos) >= length.
2177 __ Cmp(temp, OperandFrom(length, Primitive::kPrimInt));
2178 __ B(slow_path->GetEntryLabel(), lt);
2179 }
2180 } else if (length_is_input_length) {
2181 // The only way the copy can succeed is if pos is zero.
2182 __ Cbnz(WRegisterFrom(pos), slow_path->GetEntryLabel());
2183 } else {
2184 // Check that pos >= 0.
2185 Register pos_reg = WRegisterFrom(pos);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002186 __ Tbnz(pos_reg, pos_reg.GetSizeInBits() - 1, slow_path->GetEntryLabel());
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002187
2188 // Check that pos <= length(input) && (length(input) - pos) >= length.
2189 __ Ldr(temp, MemOperand(input, length_offset));
2190 __ Subs(temp, temp, pos_reg);
2191 // Ccmp if length(input) >= pos, else definitely bail to slow path (N!=V == lt).
2192 __ Ccmp(temp, OperandFrom(length, Primitive::kPrimInt), NFlag, ge);
2193 __ B(slow_path->GetEntryLabel(), lt);
2194 }
2195}
2196
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002197// Compute base source address, base destination address, and end
2198// source address for System.arraycopy* intrinsics in `src_base`,
2199// `dst_base` and `src_end` respectively.
Scott Wakeling97c72b72016-06-24 16:19:36 +01002200static void GenSystemArrayCopyAddresses(MacroAssembler* masm,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002201 Primitive::Type type,
2202 const Register& src,
2203 const Location& src_pos,
2204 const Register& dst,
2205 const Location& dst_pos,
2206 const Location& copy_length,
2207 const Register& src_base,
2208 const Register& dst_base,
2209 const Register& src_end) {
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002210 // This routine is used by the SystemArrayCopy and the SystemArrayCopyChar intrinsics.
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002211 DCHECK(type == Primitive::kPrimNot || type == Primitive::kPrimChar)
Roland Levillainebea3d22016-04-12 15:42:57 +01002212 << "Unexpected element type: " << type;
2213 const int32_t element_size = Primitive::ComponentSize(type);
2214 const int32_t element_size_shift = Primitive::ComponentSizeShift(type);
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002215 const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002216
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002217 if (src_pos.IsConstant()) {
2218 int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
Roland Levillainebea3d22016-04-12 15:42:57 +01002219 __ Add(src_base, src, element_size * constant + data_offset);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002220 } else {
Roland Levillainebea3d22016-04-12 15:42:57 +01002221 __ Add(src_base, src, data_offset);
2222 __ Add(src_base, src_base, Operand(XRegisterFrom(src_pos), LSL, element_size_shift));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002223 }
2224
2225 if (dst_pos.IsConstant()) {
2226 int32_t constant = dst_pos.GetConstant()->AsIntConstant()->GetValue();
Roland Levillainebea3d22016-04-12 15:42:57 +01002227 __ Add(dst_base, dst, element_size * constant + data_offset);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002228 } else {
Roland Levillainebea3d22016-04-12 15:42:57 +01002229 __ Add(dst_base, dst, data_offset);
2230 __ Add(dst_base, dst_base, Operand(XRegisterFrom(dst_pos), LSL, element_size_shift));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002231 }
2232
2233 if (copy_length.IsConstant()) {
2234 int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue();
Roland Levillainebea3d22016-04-12 15:42:57 +01002235 __ Add(src_end, src_base, element_size * constant);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002236 } else {
Roland Levillainebea3d22016-04-12 15:42:57 +01002237 __ Add(src_end, src_base, Operand(XRegisterFrom(copy_length), LSL, element_size_shift));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002238 }
2239}
2240
2241void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopyChar(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01002242 MacroAssembler* masm = GetVIXLAssembler();
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002243 LocationSummary* locations = invoke->GetLocations();
2244 Register src = XRegisterFrom(locations->InAt(0));
2245 Location src_pos = locations->InAt(1);
2246 Register dst = XRegisterFrom(locations->InAt(2));
2247 Location dst_pos = locations->InAt(3);
2248 Location length = locations->InAt(4);
2249
2250 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
2251 codegen_->AddSlowPath(slow_path);
2252
2253 // If source and destination are the same, take the slow path. Overlapping copy regions must be
2254 // copied in reverse and we can't know in all cases if it's needed.
2255 __ Cmp(src, dst);
2256 __ B(slow_path->GetEntryLabel(), eq);
2257
2258 // Bail out if the source is null.
2259 __ Cbz(src, slow_path->GetEntryLabel());
2260
2261 // Bail out if the destination is null.
2262 __ Cbz(dst, slow_path->GetEntryLabel());
2263
2264 if (!length.IsConstant()) {
Vladimir Markoc5646202016-11-28 16:03:15 +00002265 // Merge the following two comparisons into one:
2266 // If the length is negative, bail out (delegate to libcore's native implementation).
2267 // If the length > 32 then (currently) prefer libcore's native implementation.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002268 __ Cmp(WRegisterFrom(length), kSystemArrayCopyCharThreshold);
Vladimir Markoc5646202016-11-28 16:03:15 +00002269 __ B(slow_path->GetEntryLabel(), hi);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002270 } else {
2271 // We have already checked in the LocationsBuilder for the constant case.
2272 DCHECK_GE(length.GetConstant()->AsIntConstant()->GetValue(), 0);
2273 DCHECK_LE(length.GetConstant()->AsIntConstant()->GetValue(), 32);
2274 }
2275
2276 Register src_curr_addr = WRegisterFrom(locations->GetTemp(0));
2277 Register dst_curr_addr = WRegisterFrom(locations->GetTemp(1));
2278 Register src_stop_addr = WRegisterFrom(locations->GetTemp(2));
2279
2280 CheckSystemArrayCopyPosition(masm,
2281 src_pos,
2282 src,
2283 length,
2284 slow_path,
2285 src_curr_addr,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002286 false);
2287
2288 CheckSystemArrayCopyPosition(masm,
2289 dst_pos,
2290 dst,
2291 length,
2292 slow_path,
2293 src_curr_addr,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002294 false);
2295
2296 src_curr_addr = src_curr_addr.X();
2297 dst_curr_addr = dst_curr_addr.X();
2298 src_stop_addr = src_stop_addr.X();
2299
2300 GenSystemArrayCopyAddresses(masm,
2301 Primitive::kPrimChar,
2302 src,
2303 src_pos,
2304 dst,
2305 dst_pos,
2306 length,
2307 src_curr_addr,
2308 dst_curr_addr,
2309 src_stop_addr);
2310
2311 // Iterate over the arrays and do a raw copy of the chars.
2312 const int32_t char_size = Primitive::ComponentSize(Primitive::kPrimChar);
2313 UseScratchRegisterScope temps(masm);
2314 Register tmp = temps.AcquireW();
Scott Wakeling97c72b72016-06-24 16:19:36 +01002315 vixl::aarch64::Label loop, done;
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002316 __ Bind(&loop);
2317 __ Cmp(src_curr_addr, src_stop_addr);
2318 __ B(&done, eq);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002319 __ Ldrh(tmp, MemOperand(src_curr_addr, char_size, PostIndex));
2320 __ Strh(tmp, MemOperand(dst_curr_addr, char_size, PostIndex));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002321 __ B(&loop);
2322 __ Bind(&done);
2323
2324 __ Bind(slow_path->GetExitLabel());
2325}
2326
donghui.baic2ec9ad2016-03-10 14:02:55 +08002327// We can choose to use the native implementation there for longer copy lengths.
2328static constexpr int32_t kSystemArrayCopyThreshold = 128;
2329
2330// CodeGenerator::CreateSystemArrayCopyLocationSummary use three temporary registers.
2331// We want to use two temporary registers in order to reduce the register pressure in arm64.
2332// So we don't use the CodeGenerator::CreateSystemArrayCopyLocationSummary.
2333void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopy(HInvoke* invoke) {
Roland Levillain0b671c02016-08-19 12:02:34 +01002334 // The only read barrier implementation supporting the
2335 // SystemArrayCopy intrinsic is the Baker-style read barriers.
2336 if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) {
Roland Levillain3d312422016-06-23 13:53:42 +01002337 return;
2338 }
2339
donghui.baic2ec9ad2016-03-10 14:02:55 +08002340 // Check to see if we have known failures that will cause us to have to bail out
2341 // to the runtime, and just generate the runtime call directly.
2342 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
2343 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
2344
2345 // The positions must be non-negative.
2346 if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
2347 (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
2348 // We will have to fail anyways.
2349 return;
2350 }
2351
2352 // The length must be >= 0.
2353 HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
2354 if (length != nullptr) {
2355 int32_t len = length->GetValue();
2356 if (len < 0 || len >= kSystemArrayCopyThreshold) {
2357 // Just call as normal.
2358 return;
2359 }
2360 }
2361
2362 SystemArrayCopyOptimizations optimizations(invoke);
2363
2364 if (optimizations.GetDestinationIsSource()) {
2365 if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
2366 // We only support backward copying if source and destination are the same.
2367 return;
2368 }
2369 }
2370
2371 if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
2372 // We currently don't intrinsify primitive copying.
2373 return;
2374 }
2375
2376 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
2377 LocationSummary* locations = new (allocator) LocationSummary(invoke,
2378 LocationSummary::kCallOnSlowPath,
2379 kIntrinsified);
2380 // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
2381 locations->SetInAt(0, Location::RequiresRegister());
2382 SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1));
2383 locations->SetInAt(2, Location::RequiresRegister());
2384 SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3));
2385 SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4));
2386
2387 locations->AddTemp(Location::RequiresRegister());
2388 locations->AddTemp(Location::RequiresRegister());
Roland Levillain0b671c02016-08-19 12:02:34 +01002389 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2390 // Temporary register IP0, obtained from the VIXL scratch register
2391 // pool, cannot be used in ReadBarrierSystemArrayCopySlowPathARM64
2392 // (because that register is clobbered by ReadBarrierMarkRegX
Roland Levillain54f869e2017-03-06 13:54:11 +00002393 // entry points). It cannot be used in calls to
2394 // CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier
2395 // either. For these reasons, get a third extra temporary register
2396 // from the register allocator.
Roland Levillain0b671c02016-08-19 12:02:34 +01002397 locations->AddTemp(Location::RequiresRegister());
Roland Levillain54f869e2017-03-06 13:54:11 +00002398 } else {
2399 // Cases other than Baker read barriers: the third temporary will
2400 // be acquired from the VIXL scratch register pool.
Roland Levillain0b671c02016-08-19 12:02:34 +01002401 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002402}
2403
2404void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) {
Roland Levillain0b671c02016-08-19 12:02:34 +01002405 // The only read barrier implementation supporting the
2406 // SystemArrayCopy intrinsic is the Baker-style read barriers.
2407 DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier);
Roland Levillain3d312422016-06-23 13:53:42 +01002408
Scott Wakeling97c72b72016-06-24 16:19:36 +01002409 MacroAssembler* masm = GetVIXLAssembler();
donghui.baic2ec9ad2016-03-10 14:02:55 +08002410 LocationSummary* locations = invoke->GetLocations();
2411
2412 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2413 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2414 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2415 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
Roland Levillain0b671c02016-08-19 12:02:34 +01002416 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
donghui.baic2ec9ad2016-03-10 14:02:55 +08002417
2418 Register src = XRegisterFrom(locations->InAt(0));
2419 Location src_pos = locations->InAt(1);
2420 Register dest = XRegisterFrom(locations->InAt(2));
2421 Location dest_pos = locations->InAt(3);
2422 Location length = locations->InAt(4);
2423 Register temp1 = WRegisterFrom(locations->GetTemp(0));
Roland Levillain0b671c02016-08-19 12:02:34 +01002424 Location temp1_loc = LocationFrom(temp1);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002425 Register temp2 = WRegisterFrom(locations->GetTemp(1));
Roland Levillain0b671c02016-08-19 12:02:34 +01002426 Location temp2_loc = LocationFrom(temp2);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002427
Roland Levillain0b671c02016-08-19 12:02:34 +01002428 SlowPathCodeARM64* intrinsic_slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
2429 codegen_->AddSlowPath(intrinsic_slow_path);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002430
Scott Wakeling97c72b72016-06-24 16:19:36 +01002431 vixl::aarch64::Label conditions_on_positions_validated;
donghui.baic2ec9ad2016-03-10 14:02:55 +08002432 SystemArrayCopyOptimizations optimizations(invoke);
2433
donghui.baic2ec9ad2016-03-10 14:02:55 +08002434 // If source and destination are the same, we go to slow path if we need to do
2435 // forward copying.
2436 if (src_pos.IsConstant()) {
2437 int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
2438 if (dest_pos.IsConstant()) {
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002439 int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
2440 if (optimizations.GetDestinationIsSource()) {
2441 // Checked when building locations.
2442 DCHECK_GE(src_pos_constant, dest_pos_constant);
2443 } else if (src_pos_constant < dest_pos_constant) {
2444 __ Cmp(src, dest);
Roland Levillain0b671c02016-08-19 12:02:34 +01002445 __ B(intrinsic_slow_path->GetEntryLabel(), eq);
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002446 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002447 // Checked when building locations.
2448 DCHECK(!optimizations.GetDestinationIsSource()
2449 || (src_pos_constant >= dest_pos.GetConstant()->AsIntConstant()->GetValue()));
2450 } else {
2451 if (!optimizations.GetDestinationIsSource()) {
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002452 __ Cmp(src, dest);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002453 __ B(&conditions_on_positions_validated, ne);
2454 }
2455 __ Cmp(WRegisterFrom(dest_pos), src_pos_constant);
Roland Levillain0b671c02016-08-19 12:02:34 +01002456 __ B(intrinsic_slow_path->GetEntryLabel(), gt);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002457 }
2458 } else {
2459 if (!optimizations.GetDestinationIsSource()) {
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002460 __ Cmp(src, dest);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002461 __ B(&conditions_on_positions_validated, ne);
2462 }
2463 __ Cmp(RegisterFrom(src_pos, invoke->InputAt(1)->GetType()),
2464 OperandFrom(dest_pos, invoke->InputAt(3)->GetType()));
Roland Levillain0b671c02016-08-19 12:02:34 +01002465 __ B(intrinsic_slow_path->GetEntryLabel(), lt);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002466 }
2467
2468 __ Bind(&conditions_on_positions_validated);
2469
2470 if (!optimizations.GetSourceIsNotNull()) {
2471 // Bail out if the source is null.
Roland Levillain0b671c02016-08-19 12:02:34 +01002472 __ Cbz(src, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002473 }
2474
2475 if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) {
2476 // Bail out if the destination is null.
Roland Levillain0b671c02016-08-19 12:02:34 +01002477 __ Cbz(dest, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002478 }
2479
2480 // We have already checked in the LocationsBuilder for the constant case.
2481 if (!length.IsConstant() &&
2482 !optimizations.GetCountIsSourceLength() &&
2483 !optimizations.GetCountIsDestinationLength()) {
Vladimir Markoc5646202016-11-28 16:03:15 +00002484 // Merge the following two comparisons into one:
2485 // If the length is negative, bail out (delegate to libcore's native implementation).
2486 // If the length >= 128 then (currently) prefer native implementation.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002487 __ Cmp(WRegisterFrom(length), kSystemArrayCopyThreshold);
Vladimir Markoc5646202016-11-28 16:03:15 +00002488 __ B(intrinsic_slow_path->GetEntryLabel(), hs);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002489 }
2490 // Validity checks: source.
2491 CheckSystemArrayCopyPosition(masm,
2492 src_pos,
2493 src,
2494 length,
Roland Levillain0b671c02016-08-19 12:02:34 +01002495 intrinsic_slow_path,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002496 temp1,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002497 optimizations.GetCountIsSourceLength());
2498
2499 // Validity checks: dest.
2500 CheckSystemArrayCopyPosition(masm,
2501 dest_pos,
2502 dest,
2503 length,
Roland Levillain0b671c02016-08-19 12:02:34 +01002504 intrinsic_slow_path,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002505 temp1,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002506 optimizations.GetCountIsDestinationLength());
2507 {
2508 // We use a block to end the scratch scope before the write barrier, thus
2509 // freeing the temporary registers so they can be used in `MarkGCCard`.
2510 UseScratchRegisterScope temps(masm);
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002511 Location temp3_loc; // Used only for Baker read barrier.
Roland Levillain54f869e2017-03-06 13:54:11 +00002512 Register temp3;
2513 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002514 temp3_loc = locations->GetTemp(2);
2515 temp3 = WRegisterFrom(temp3_loc);
Roland Levillain54f869e2017-03-06 13:54:11 +00002516 } else {
2517 temp3 = temps.AcquireW();
2518 }
Roland Levillain0b671c02016-08-19 12:02:34 +01002519
donghui.baic2ec9ad2016-03-10 14:02:55 +08002520 if (!optimizations.GetDoesNotNeedTypeCheck()) {
2521 // Check whether all elements of the source array are assignable to the component
2522 // type of the destination array. We do two checks: the classes are the same,
2523 // or the destination is Object[]. If none of these checks succeed, we go to the
2524 // slow path.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002525
Roland Levillain0b671c02016-08-19 12:02:34 +01002526 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2527 if (!optimizations.GetSourceIsNonPrimitiveArray()) {
2528 // /* HeapReference<Class> */ temp1 = src->klass_
2529 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2530 temp1_loc,
2531 src.W(),
2532 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002533 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002534 /* needs_null_check */ false,
2535 /* use_load_acquire */ false);
2536 // Bail out if the source is not a non primitive array.
2537 // /* HeapReference<Class> */ temp1 = temp1->component_type_
2538 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2539 temp1_loc,
2540 temp1,
2541 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002542 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002543 /* needs_null_check */ false,
2544 /* use_load_acquire */ false);
2545 __ Cbz(temp1, intrinsic_slow_path->GetEntryLabel());
2546 // If heap poisoning is enabled, `temp1` has been unpoisoned
2547 // by the the previous call to GenerateFieldLoadWithBakerReadBarrier.
2548 // /* uint16_t */ temp1 = static_cast<uint16>(temp1->primitive_type_);
2549 __ Ldrh(temp1, HeapOperand(temp1, primitive_offset));
2550 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2551 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002552 }
Roland Levillain0b671c02016-08-19 12:02:34 +01002553
2554 // /* HeapReference<Class> */ temp1 = dest->klass_
2555 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2556 temp1_loc,
2557 dest.W(),
2558 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002559 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002560 /* needs_null_check */ false,
2561 /* use_load_acquire */ false);
2562
2563 if (!optimizations.GetDestinationIsNonPrimitiveArray()) {
2564 // Bail out if the destination is not a non primitive array.
2565 //
2566 // Register `temp1` is not trashed by the read barrier emitted
2567 // by GenerateFieldLoadWithBakerReadBarrier below, as that
2568 // method produces a call to a ReadBarrierMarkRegX entry point,
2569 // which saves all potentially live registers, including
2570 // temporaries such a `temp1`.
2571 // /* HeapReference<Class> */ temp2 = temp1->component_type_
2572 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2573 temp2_loc,
2574 temp1,
2575 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002576 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002577 /* needs_null_check */ false,
2578 /* use_load_acquire */ false);
2579 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel());
2580 // If heap poisoning is enabled, `temp2` has been unpoisoned
2581 // by the the previous call to GenerateFieldLoadWithBakerReadBarrier.
2582 // /* uint16_t */ temp2 = static_cast<uint16>(temp2->primitive_type_);
2583 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset));
2584 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2585 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel());
2586 }
2587
2588 // For the same reason given earlier, `temp1` is not trashed by the
2589 // read barrier emitted by GenerateFieldLoadWithBakerReadBarrier below.
2590 // /* HeapReference<Class> */ temp2 = src->klass_
2591 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2592 temp2_loc,
2593 src.W(),
2594 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002595 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002596 /* needs_null_check */ false,
2597 /* use_load_acquire */ false);
2598 // Note: if heap poisoning is on, we are comparing two unpoisoned references here.
2599 __ Cmp(temp1, temp2);
2600
2601 if (optimizations.GetDestinationIsTypedObjectArray()) {
2602 vixl::aarch64::Label do_copy;
2603 __ B(&do_copy, eq);
2604 // /* HeapReference<Class> */ temp1 = temp1->component_type_
2605 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2606 temp1_loc,
2607 temp1,
2608 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002609 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002610 /* needs_null_check */ false,
2611 /* use_load_acquire */ false);
2612 // /* HeapReference<Class> */ temp1 = temp1->super_class_
2613 // We do not need to emit a read barrier for the following
2614 // heap reference load, as `temp1` is only used in a
2615 // comparison with null below, and this reference is not
2616 // kept afterwards.
2617 __ Ldr(temp1, HeapOperand(temp1, super_offset));
2618 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel());
2619 __ Bind(&do_copy);
2620 } else {
2621 __ B(intrinsic_slow_path->GetEntryLabel(), ne);
2622 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002623 } else {
Roland Levillain0b671c02016-08-19 12:02:34 +01002624 // Non read barrier code.
2625
2626 // /* HeapReference<Class> */ temp1 = dest->klass_
2627 __ Ldr(temp1, MemOperand(dest, class_offset));
2628 // /* HeapReference<Class> */ temp2 = src->klass_
2629 __ Ldr(temp2, MemOperand(src, class_offset));
2630 bool did_unpoison = false;
2631 if (!optimizations.GetDestinationIsNonPrimitiveArray() ||
2632 !optimizations.GetSourceIsNonPrimitiveArray()) {
2633 // One or two of the references need to be unpoisoned. Unpoison them
2634 // both to make the identity check valid.
2635 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2636 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
2637 did_unpoison = true;
2638 }
2639
2640 if (!optimizations.GetDestinationIsNonPrimitiveArray()) {
2641 // Bail out if the destination is not a non primitive array.
2642 // /* HeapReference<Class> */ temp3 = temp1->component_type_
2643 __ Ldr(temp3, HeapOperand(temp1, component_offset));
2644 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel());
2645 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp3);
2646 // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_);
2647 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset));
2648 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2649 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel());
2650 }
2651
2652 if (!optimizations.GetSourceIsNonPrimitiveArray()) {
2653 // Bail out if the source is not a non primitive array.
2654 // /* HeapReference<Class> */ temp3 = temp2->component_type_
2655 __ Ldr(temp3, HeapOperand(temp2, component_offset));
2656 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel());
2657 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp3);
2658 // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_);
2659 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset));
2660 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2661 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel());
2662 }
2663
2664 __ Cmp(temp1, temp2);
2665
2666 if (optimizations.GetDestinationIsTypedObjectArray()) {
2667 vixl::aarch64::Label do_copy;
2668 __ B(&do_copy, eq);
2669 if (!did_unpoison) {
2670 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2671 }
2672 // /* HeapReference<Class> */ temp1 = temp1->component_type_
2673 __ Ldr(temp1, HeapOperand(temp1, component_offset));
2674 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2675 // /* HeapReference<Class> */ temp1 = temp1->super_class_
2676 __ Ldr(temp1, HeapOperand(temp1, super_offset));
2677 // No need to unpoison the result, we're comparing against null.
2678 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel());
2679 __ Bind(&do_copy);
2680 } else {
2681 __ B(intrinsic_slow_path->GetEntryLabel(), ne);
2682 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002683 }
2684 } else if (!optimizations.GetSourceIsNonPrimitiveArray()) {
2685 DCHECK(optimizations.GetDestinationIsNonPrimitiveArray());
2686 // Bail out if the source is not a non primitive array.
Roland Levillain0b671c02016-08-19 12:02:34 +01002687 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2688 // /* HeapReference<Class> */ temp1 = src->klass_
2689 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2690 temp1_loc,
2691 src.W(),
2692 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002693 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002694 /* needs_null_check */ false,
2695 /* use_load_acquire */ false);
2696 // /* HeapReference<Class> */ temp2 = temp1->component_type_
2697 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2698 temp2_loc,
2699 temp1,
2700 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002701 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002702 /* needs_null_check */ false,
2703 /* use_load_acquire */ false);
2704 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel());
2705 // If heap poisoning is enabled, `temp2` has been unpoisoned
2706 // by the the previous call to GenerateFieldLoadWithBakerReadBarrier.
2707 } else {
2708 // /* HeapReference<Class> */ temp1 = src->klass_
2709 __ Ldr(temp1, HeapOperand(src.W(), class_offset));
2710 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2711 // /* HeapReference<Class> */ temp2 = temp1->component_type_
2712 __ Ldr(temp2, HeapOperand(temp1, component_offset));
2713 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel());
2714 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
2715 }
2716 // /* uint16_t */ temp2 = static_cast<uint16>(temp2->primitive_type_);
2717 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset));
donghui.baic2ec9ad2016-03-10 14:02:55 +08002718 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
Roland Levillain0b671c02016-08-19 12:02:34 +01002719 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002720 }
2721
Roland Levillain1663d162017-03-17 15:15:21 +00002722 if (length.IsConstant() && length.GetConstant()->AsIntConstant()->GetValue() == 0) {
2723 // Null constant length: not need to emit the loop code at all.
Roland Levillain0b671c02016-08-19 12:02:34 +01002724 } else {
Roland Levillain1663d162017-03-17 15:15:21 +00002725 Register src_curr_addr = temp1.X();
2726 Register dst_curr_addr = temp2.X();
2727 Register src_stop_addr = temp3.X();
2728 vixl::aarch64::Label done;
2729 const Primitive::Type type = Primitive::kPrimNot;
2730 const int32_t element_size = Primitive::ComponentSize(type);
2731
2732 if (length.IsRegister()) {
2733 // Don't enter the copy loop if the length is null.
2734 __ Cbz(WRegisterFrom(length), &done);
2735 }
2736
2737 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2738 // TODO: Also convert this intrinsic to the IsGcMarking strategy?
2739
2740 // SystemArrayCopy implementation for Baker read barriers (see
Roland Levillain9983e302017-07-14 14:34:22 +01002741 // also CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier):
Roland Levillain1663d162017-03-17 15:15:21 +00002742 //
2743 // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState();
2744 // lfence; // Load fence or artificial data dependency to prevent load-load reordering
2745 // bool is_gray = (rb_state == ReadBarrier::GrayState());
2746 // if (is_gray) {
2747 // // Slow-path copy.
2748 // do {
2749 // *dest_ptr++ = MaybePoison(ReadBarrier::Mark(MaybeUnpoison(*src_ptr++)));
2750 // } while (src_ptr != end_ptr)
2751 // } else {
2752 // // Fast-path copy.
2753 // do {
2754 // *dest_ptr++ = *src_ptr++;
2755 // } while (src_ptr != end_ptr)
2756 // }
2757
2758 // Make sure `tmp` is not IP0, as it is clobbered by
2759 // ReadBarrierMarkRegX entry points in
2760 // ReadBarrierSystemArrayCopySlowPathARM64.
Roland Levillain1ca955d2017-04-13 19:34:30 +01002761 DCHECK(temps.IsAvailable(ip0));
Roland Levillain1663d162017-03-17 15:15:21 +00002762 temps.Exclude(ip0);
Roland Levillain0b671c02016-08-19 12:02:34 +01002763 Register tmp = temps.AcquireW();
Roland Levillain1663d162017-03-17 15:15:21 +00002764 DCHECK_NE(LocationFrom(tmp).reg(), IP0);
Roland Levillain1ca955d2017-04-13 19:34:30 +01002765 // Put IP0 back in the pool so that VIXL has at least one
2766 // scratch register available to emit macro-instructions (note
2767 // that IP1 is already used for `tmp`). Indeed some
2768 // macro-instructions used in GenSystemArrayCopyAddresses
2769 // (invoked hereunder) may require a scratch register (for
2770 // instance to emit a load with a large constant offset).
2771 temps.Include(ip0);
Roland Levillain1663d162017-03-17 15:15:21 +00002772
2773 // /* int32_t */ monitor = src->monitor_
2774 __ Ldr(tmp, HeapOperand(src.W(), monitor_offset));
2775 // /* LockWord */ lock_word = LockWord(monitor)
2776 static_assert(sizeof(LockWord) == sizeof(int32_t),
2777 "art::LockWord and int32_t have different sizes.");
2778
2779 // Introduce a dependency on the lock_word including rb_state,
2780 // to prevent load-load reordering, and without using
2781 // a memory barrier (which would be more expensive).
2782 // `src` is unchanged by this operation, but its value now depends
2783 // on `tmp`.
2784 __ Add(src.X(), src.X(), Operand(tmp.X(), LSR, 32));
2785
2786 // Compute base source address, base destination address, and end
2787 // source address for System.arraycopy* intrinsics in `src_base`,
2788 // `dst_base` and `src_end` respectively.
2789 // Note that `src_curr_addr` is computed from from `src` (and
2790 // `src_pos`) here, and thus honors the artificial dependency
2791 // of `src` on `tmp`.
2792 GenSystemArrayCopyAddresses(masm,
2793 type,
2794 src,
2795 src_pos,
2796 dest,
2797 dest_pos,
2798 length,
2799 src_curr_addr,
2800 dst_curr_addr,
2801 src_stop_addr);
2802
2803 // Slow path used to copy array when `src` is gray.
2804 SlowPathCodeARM64* read_barrier_slow_path =
2805 new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathARM64(invoke, LocationFrom(tmp));
2806 codegen_->AddSlowPath(read_barrier_slow_path);
2807
2808 // Given the numeric representation, it's enough to check the low bit of the rb_state.
2809 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
2810 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
2811 __ Tbnz(tmp, LockWord::kReadBarrierStateShift, read_barrier_slow_path->GetEntryLabel());
2812
2813 // Fast-path copy.
2814 // Iterate over the arrays and do a raw copy of the objects. We don't need to
2815 // poison/unpoison.
2816 vixl::aarch64::Label loop;
2817 __ Bind(&loop);
Roland Levillain0b671c02016-08-19 12:02:34 +01002818 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex));
2819 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex));
Roland Levillain1663d162017-03-17 15:15:21 +00002820 __ Cmp(src_curr_addr, src_stop_addr);
2821 __ B(&loop, ne);
2822
2823 __ Bind(read_barrier_slow_path->GetExitLabel());
2824 } else {
2825 // Non read barrier code.
2826 // Compute base source address, base destination address, and end
2827 // source address for System.arraycopy* intrinsics in `src_base`,
2828 // `dst_base` and `src_end` respectively.
2829 GenSystemArrayCopyAddresses(masm,
2830 type,
2831 src,
2832 src_pos,
2833 dest,
2834 dest_pos,
2835 length,
2836 src_curr_addr,
2837 dst_curr_addr,
2838 src_stop_addr);
2839 // Iterate over the arrays and do a raw copy of the objects. We don't need to
2840 // poison/unpoison.
2841 vixl::aarch64::Label loop;
2842 __ Bind(&loop);
2843 {
2844 Register tmp = temps.AcquireW();
2845 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex));
2846 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex));
2847 }
2848 __ Cmp(src_curr_addr, src_stop_addr);
2849 __ B(&loop, ne);
Roland Levillain0b671c02016-08-19 12:02:34 +01002850 }
Roland Levillain0b671c02016-08-19 12:02:34 +01002851 __ Bind(&done);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002852 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002853 }
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002854
donghui.baic2ec9ad2016-03-10 14:02:55 +08002855 // We only need one card marking on the destination array.
2856 codegen_->MarkGCCard(dest.W(), Register(), /* value_can_be_null */ false);
2857
Roland Levillain0b671c02016-08-19 12:02:34 +01002858 __ Bind(intrinsic_slow_path->GetExitLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002859}
2860
Anton Kirilova3ffea22016-04-07 17:02:37 +01002861static void GenIsInfinite(LocationSummary* locations,
2862 bool is64bit,
Scott Wakeling97c72b72016-06-24 16:19:36 +01002863 MacroAssembler* masm) {
Anton Kirilova3ffea22016-04-07 17:02:37 +01002864 Operand infinity;
2865 Register out;
2866
2867 if (is64bit) {
2868 infinity = kPositiveInfinityDouble;
2869 out = XRegisterFrom(locations->Out());
2870 } else {
2871 infinity = kPositiveInfinityFloat;
2872 out = WRegisterFrom(locations->Out());
2873 }
2874
Scott Wakeling97c72b72016-06-24 16:19:36 +01002875 const Register zero = vixl::aarch64::Assembler::AppropriateZeroRegFor(out);
Anton Kirilova3ffea22016-04-07 17:02:37 +01002876
2877 MoveFPToInt(locations, is64bit, masm);
2878 __ Eor(out, out, infinity);
2879 // We don't care about the sign bit, so shift left.
2880 __ Cmp(zero, Operand(out, LSL, 1));
2881 __ Cset(out, eq);
2882}
2883
2884void IntrinsicLocationsBuilderARM64::VisitFloatIsInfinite(HInvoke* invoke) {
2885 CreateFPToIntLocations(arena_, invoke);
2886}
2887
2888void IntrinsicCodeGeneratorARM64::VisitFloatIsInfinite(HInvoke* invoke) {
2889 GenIsInfinite(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
2890}
2891
2892void IntrinsicLocationsBuilderARM64::VisitDoubleIsInfinite(HInvoke* invoke) {
2893 CreateFPToIntLocations(arena_, invoke);
2894}
2895
2896void IntrinsicCodeGeneratorARM64::VisitDoubleIsInfinite(HInvoke* invoke) {
2897 GenIsInfinite(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
2898}
2899
Nicolas Geoffray331605a2017-03-01 11:01:41 +00002900void IntrinsicLocationsBuilderARM64::VisitIntegerValueOf(HInvoke* invoke) {
2901 InvokeRuntimeCallingConvention calling_convention;
2902 IntrinsicVisitor::ComputeIntegerValueOfLocations(
2903 invoke,
2904 codegen_,
2905 calling_convention.GetReturnLocation(Primitive::kPrimNot),
2906 Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode()));
2907}
2908
2909void IntrinsicCodeGeneratorARM64::VisitIntegerValueOf(HInvoke* invoke) {
2910 IntrinsicVisitor::IntegerValueOfInfo info = IntrinsicVisitor::ComputeIntegerValueOfInfo();
2911 LocationSummary* locations = invoke->GetLocations();
2912 MacroAssembler* masm = GetVIXLAssembler();
2913
2914 Register out = RegisterFrom(locations->Out(), Primitive::kPrimNot);
2915 UseScratchRegisterScope temps(masm);
2916 Register temp = temps.AcquireW();
2917 InvokeRuntimeCallingConvention calling_convention;
2918 Register argument = calling_convention.GetRegisterAt(0);
2919 if (invoke->InputAt(0)->IsConstant()) {
2920 int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue();
2921 if (value >= info.low && value <= info.high) {
2922 // Just embed the j.l.Integer in the code.
2923 ScopedObjectAccess soa(Thread::Current());
2924 mirror::Object* boxed = info.cache->Get(value + (-info.low));
2925 DCHECK(boxed != nullptr && Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(boxed));
2926 uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(boxed));
2927 __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(address));
2928 } else {
2929 // Allocate and initialize a new j.l.Integer.
2930 // TODO: If we JIT, we could allocate the j.l.Integer now, and store it in the
2931 // JIT object table.
2932 uint32_t address =
2933 dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer));
2934 __ Ldr(argument.W(), codegen_->DeduplicateBootImageAddressLiteral(address));
2935 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc());
2936 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
2937 __ Mov(temp.W(), value);
2938 __ Str(temp.W(), HeapOperand(out.W(), info.value_offset));
2939 // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation
2940 // one.
2941 codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore);
2942 }
2943 } else {
2944 Register in = RegisterFrom(locations->InAt(0), Primitive::kPrimInt);
2945 // Check bounds of our cache.
2946 __ Add(out.W(), in.W(), -info.low);
2947 __ Cmp(out.W(), info.high - info.low + 1);
2948 vixl::aarch64::Label allocate, done;
2949 __ B(&allocate, hs);
2950 // If the value is within the bounds, load the j.l.Integer directly from the array.
2951 uint32_t data_offset = mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value();
2952 uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.cache));
2953 __ Ldr(temp.W(), codegen_->DeduplicateBootImageAddressLiteral(data_offset + address));
2954 MemOperand source = HeapOperand(
2955 temp, out.X(), LSL, Primitive::ComponentSizeShift(Primitive::kPrimNot));
2956 codegen_->Load(Primitive::kPrimNot, out, source);
2957 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(out);
2958 __ B(&done);
2959 __ Bind(&allocate);
2960 // Otherwise allocate and initialize a new j.l.Integer.
2961 address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer));
2962 __ Ldr(argument.W(), codegen_->DeduplicateBootImageAddressLiteral(address));
2963 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc());
2964 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
2965 __ Str(in.W(), HeapOperand(out.W(), info.value_offset));
2966 // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation
2967 // one.
2968 codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore);
2969 __ Bind(&done);
2970 }
2971}
2972
Nicolas Geoffray365719c2017-03-08 13:11:50 +00002973void IntrinsicLocationsBuilderARM64::VisitThreadInterrupted(HInvoke* invoke) {
2974 LocationSummary* locations = new (arena_) LocationSummary(invoke,
2975 LocationSummary::kNoCall,
2976 kIntrinsified);
2977 locations->SetOut(Location::RequiresRegister());
2978}
2979
2980void IntrinsicCodeGeneratorARM64::VisitThreadInterrupted(HInvoke* invoke) {
2981 MacroAssembler* masm = GetVIXLAssembler();
2982 Register out = RegisterFrom(invoke->GetLocations()->Out(), Primitive::kPrimInt);
2983 UseScratchRegisterScope temps(masm);
2984 Register temp = temps.AcquireX();
2985
2986 __ Add(temp, tr, Thread::InterruptedOffset<kArm64PointerSize>().Int32Value());
2987 __ Ldar(out.W(), MemOperand(temp));
2988
2989 vixl::aarch64::Label done;
2990 __ Cbz(out.W(), &done);
2991 __ Stlr(wzr, MemOperand(temp));
2992 __ Bind(&done);
2993}
2994
Vladimir Markod254f5c2017-06-02 15:18:36 +00002995UNIMPLEMENTED_INTRINSIC(ARM64, ReferenceGetReferent)
Aart Bik2f9fcc92016-03-01 15:16:54 -08002996UNIMPLEMENTED_INTRINSIC(ARM64, IntegerHighestOneBit)
2997UNIMPLEMENTED_INTRINSIC(ARM64, LongHighestOneBit)
2998UNIMPLEMENTED_INTRINSIC(ARM64, IntegerLowestOneBit)
2999UNIMPLEMENTED_INTRINSIC(ARM64, LongLowestOneBit)
Andreas Gampe878d58c2015-01-15 23:24:00 -08003000
Aart Bikff7d89c2016-11-07 08:49:28 -08003001UNIMPLEMENTED_INTRINSIC(ARM64, StringStringIndexOf);
3002UNIMPLEMENTED_INTRINSIC(ARM64, StringStringIndexOfAfter);
Aart Bik71bf7b42016-11-16 10:17:46 -08003003UNIMPLEMENTED_INTRINSIC(ARM64, StringBufferAppend);
3004UNIMPLEMENTED_INTRINSIC(ARM64, StringBufferLength);
3005UNIMPLEMENTED_INTRINSIC(ARM64, StringBufferToString);
3006UNIMPLEMENTED_INTRINSIC(ARM64, StringBuilderAppend);
3007UNIMPLEMENTED_INTRINSIC(ARM64, StringBuilderLength);
3008UNIMPLEMENTED_INTRINSIC(ARM64, StringBuilderToString);
Aart Bikff7d89c2016-11-07 08:49:28 -08003009
Aart Bik0e54c012016-03-04 12:08:31 -08003010// 1.8.
3011UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndAddInt)
3012UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndAddLong)
3013UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetInt)
3014UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetLong)
3015UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetObject)
Aart Bik0e54c012016-03-04 12:08:31 -08003016
Aart Bik2f9fcc92016-03-01 15:16:54 -08003017UNREACHABLE_INTRINSICS(ARM64)
Roland Levillain4d027112015-07-01 15:41:14 +01003018
3019#undef __
3020
Andreas Gampe878d58c2015-01-15 23:24:00 -08003021} // namespace arm64
3022} // namespace art