blob: ee07c4f65cf2982d94eea62b8cb0233ef00b69ca [file] [log] [blame]
Andreas Gampe878d58c2015-01-15 23:24:00 -08001/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
Serban Constantinescu579885a2015-02-22 20:51:33 +000019#include "arch/arm64/instruction_set_features_arm64.h"
Mathieu Chartiere401d142015-04-22 13:56:20 -070020#include "art_method.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080021#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
Andreas Gampe09659c22017-09-18 18:23:32 -070024#include "heap_poisoning.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080025#include "intrinsics.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080026#include "lock_word.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080027#include "mirror/array-inl.h"
Andreas Gampec15a2f42017-04-21 12:09:39 -070028#include "mirror/object_array-inl.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080029#include "mirror/reference.h"
Vladimir Markoe39f14f2017-02-10 15:44:25 +000030#include "mirror/string-inl.h"
Andreas Gampec6ea7d02017-02-01 16:46:28 -080031#include "scoped_thread_state_change-inl.h"
Andreas Gampeb486a982017-06-01 13:45:54 -070032#include "thread-current-inl.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080033#include "utils/arm64/assembler_arm64.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080034
Scott Wakeling97c72b72016-06-24 16:19:36 +010035using namespace vixl::aarch64; // NOLINT(build/namespaces)
Andreas Gampe878d58c2015-01-15 23:24:00 -080036
Artem Serovaf4e42a2016-08-08 15:11:24 +010037// TODO(VIXL): Make VIXL compile with -Wshadow.
Scott Wakeling97c72b72016-06-24 16:19:36 +010038#pragma GCC diagnostic push
39#pragma GCC diagnostic ignored "-Wshadow"
Artem Serovaf4e42a2016-08-08 15:11:24 +010040#include "aarch64/disasm-aarch64.h"
41#include "aarch64/macro-assembler-aarch64.h"
Scott Wakeling97c72b72016-06-24 16:19:36 +010042#pragma GCC diagnostic pop
Andreas Gampe878d58c2015-01-15 23:24:00 -080043
44namespace art {
45
46namespace arm64 {
47
48using helpers::DRegisterFrom;
49using helpers::FPRegisterFrom;
50using helpers::HeapOperand;
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +000051using helpers::LocationFrom;
Scott Wakeling9ee23f42015-07-23 10:44:35 +010052using helpers::OperandFrom;
Andreas Gampe878d58c2015-01-15 23:24:00 -080053using helpers::RegisterFrom;
54using helpers::SRegisterFrom;
55using helpers::WRegisterFrom;
56using helpers::XRegisterFrom;
xueliang.zhong49924c92016-03-03 10:52:51 +000057using helpers::InputRegisterAt;
Scott Wakeling1f36f412016-04-21 11:13:45 +010058using helpers::OutputRegister;
Andreas Gampe878d58c2015-01-15 23:24:00 -080059
Andreas Gampe878d58c2015-01-15 23:24:00 -080060namespace {
61
62ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
63 return MemOperand(XRegisterFrom(location), offset);
64}
65
66} // namespace
67
Scott Wakeling97c72b72016-06-24 16:19:36 +010068MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
Alexandre Rames087930f2016-08-02 13:45:28 +010069 return codegen_->GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -080070}
71
72ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
Vladimir Markoca6fff82017-10-03 14:49:14 +010073 return codegen_->GetGraph()->GetAllocator();
Andreas Gampe878d58c2015-01-15 23:24:00 -080074}
75
Alexandre Rames087930f2016-08-02 13:45:28 +010076#define __ codegen->GetVIXLAssembler()->
Andreas Gampe878d58c2015-01-15 23:24:00 -080077
78static void MoveFromReturnRegister(Location trg,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010079 DataType::Type type,
Andreas Gampe878d58c2015-01-15 23:24:00 -080080 CodeGeneratorARM64* codegen) {
81 if (!trg.IsValid()) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010082 DCHECK(type == DataType::Type::kVoid);
Andreas Gampe878d58c2015-01-15 23:24:00 -080083 return;
84 }
85
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010086 DCHECK_NE(type, DataType::Type::kVoid);
Andreas Gampe878d58c2015-01-15 23:24:00 -080087
Vladimir Marko0ebe0d82017-09-21 22:50:39 +010088 if (DataType::IsIntegralType(type) || type == DataType::Type::kReference) {
Andreas Gampe878d58c2015-01-15 23:24:00 -080089 Register trg_reg = RegisterFrom(trg, type);
90 Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
91 __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
92 } else {
93 FPRegister trg_reg = FPRegisterFrom(trg, type);
94 FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
95 __ Fmov(trg_reg, res_reg);
96 }
97}
98
Roland Levillainec525fc2015-04-28 15:50:20 +010099static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
Roland Levillain2d27c8e2015-04-28 15:48:45 +0100100 InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
Roland Levillainec525fc2015-04-28 15:50:20 +0100101 IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800102}
103
104// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
105// call. This will copy the arguments into the positions for a regular call.
106//
107// Note: The actual parameters are required to be in the locations given by the invoke's location
108// summary. If an intrinsic modifies those locations before a slowpath call, they must be
109// restored!
110class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
111 public:
David Srbecky9cd6d372016-02-09 15:24:47 +0000112 explicit IntrinsicSlowPathARM64(HInvoke* invoke)
113 : SlowPathCodeARM64(invoke), invoke_(invoke) { }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800114
115 void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
116 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
117 __ Bind(GetEntryLabel());
118
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000119 SaveLiveRegisters(codegen, invoke_->GetLocations());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800120
Roland Levillainec525fc2015-04-28 15:50:20 +0100121 MoveArguments(invoke_, codegen);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800122
Artem Serov914d7a82017-02-07 14:33:49 +0000123 {
124 // Ensure that between the BLR (emitted by Generate*Call) and RecordPcInfo there
125 // are no pools emitted.
126 vixl::EmissionCheckScope guard(codegen->GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
127 if (invoke_->IsInvokeStaticOrDirect()) {
Vladimir Markoe7197bf2017-06-02 17:00:23 +0100128 codegen->GenerateStaticOrDirectCall(
129 invoke_->AsInvokeStaticOrDirect(), LocationFrom(kArtMethodRegister), this);
Artem Serov914d7a82017-02-07 14:33:49 +0000130 } else {
Vladimir Markoe7197bf2017-06-02 17:00:23 +0100131 codegen->GenerateVirtualCall(
132 invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister), this);
Artem Serov914d7a82017-02-07 14:33:49 +0000133 }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800134 }
135
136 // Copy the result back to the expected output.
137 Location out = invoke_->GetLocations()->Out();
138 if (out.IsValid()) {
139 DCHECK(out.IsRegister()); // TODO: Replace this when we support output in memory.
140 DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
141 MoveFromReturnRegister(out, invoke_->GetType(), codegen);
142 }
143
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000144 RestoreLiveRegisters(codegen, invoke_->GetLocations());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800145 __ B(GetExitLabel());
146 }
147
Alexandre Rames9931f312015-06-19 14:47:01 +0100148 const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
149
Andreas Gampe878d58c2015-01-15 23:24:00 -0800150 private:
151 // The instruction where this slow path is happening.
152 HInvoke* const invoke_;
153
154 DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
155};
156
Roland Levillain0b671c02016-08-19 12:02:34 +0100157// Slow path implementing the SystemArrayCopy intrinsic copy loop with read barriers.
158class ReadBarrierSystemArrayCopySlowPathARM64 : public SlowPathCodeARM64 {
159 public:
160 ReadBarrierSystemArrayCopySlowPathARM64(HInstruction* instruction, Location tmp)
161 : SlowPathCodeARM64(instruction), tmp_(tmp) {
162 DCHECK(kEmitCompilerReadBarrier);
163 DCHECK(kUseBakerReadBarrier);
164 }
165
166 void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
167 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
168 LocationSummary* locations = instruction_->GetLocations();
169 DCHECK(locations->CanCall());
170 DCHECK(instruction_->IsInvokeStaticOrDirect())
171 << "Unexpected instruction in read barrier arraycopy slow path: "
172 << instruction_->DebugName();
173 DCHECK(instruction_->GetLocations()->Intrinsified());
174 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy);
175
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100176 const int32_t element_size = DataType::Size(DataType::Type::kReference);
Roland Levillain0b671c02016-08-19 12:02:34 +0100177
178 Register src_curr_addr = XRegisterFrom(locations->GetTemp(0));
179 Register dst_curr_addr = XRegisterFrom(locations->GetTemp(1));
180 Register src_stop_addr = XRegisterFrom(locations->GetTemp(2));
181 Register tmp_reg = WRegisterFrom(tmp_);
182
183 __ Bind(GetEntryLabel());
184 vixl::aarch64::Label slow_copy_loop;
185 __ Bind(&slow_copy_loop);
186 __ Ldr(tmp_reg, MemOperand(src_curr_addr, element_size, PostIndex));
187 codegen->GetAssembler()->MaybeUnpoisonHeapReference(tmp_reg);
188 // TODO: Inline the mark bit check before calling the runtime?
189 // tmp_reg = ReadBarrier::Mark(tmp_reg);
190 // No need to save live registers; it's taken care of by the
191 // entrypoint. Also, there is no need to update the stack mask,
192 // as this runtime call will not trigger a garbage collection.
193 // (See ReadBarrierMarkSlowPathARM64::EmitNativeCode for more
194 // explanations.)
195 DCHECK_NE(tmp_.reg(), LR);
196 DCHECK_NE(tmp_.reg(), WSP);
197 DCHECK_NE(tmp_.reg(), WZR);
198 // IP0 is used internally by the ReadBarrierMarkRegX entry point
199 // as a temporary (and not preserved). It thus cannot be used by
200 // any live register in this slow path.
201 DCHECK_NE(LocationFrom(src_curr_addr).reg(), IP0);
202 DCHECK_NE(LocationFrom(dst_curr_addr).reg(), IP0);
203 DCHECK_NE(LocationFrom(src_stop_addr).reg(), IP0);
204 DCHECK_NE(tmp_.reg(), IP0);
205 DCHECK(0 <= tmp_.reg() && tmp_.reg() < kNumberOfWRegisters) << tmp_.reg();
Roland Levillain9cc0ea82017-03-16 11:25:59 +0000206 // TODO: Load the entrypoint once before the loop, instead of
207 // loading it at every iteration.
Roland Levillain0b671c02016-08-19 12:02:34 +0100208 int32_t entry_point_offset =
Roland Levillain97c46462017-05-11 14:04:03 +0100209 Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg());
Roland Levillain0b671c02016-08-19 12:02:34 +0100210 // This runtime call does not require a stack map.
211 codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
212 codegen->GetAssembler()->MaybePoisonHeapReference(tmp_reg);
213 __ Str(tmp_reg, MemOperand(dst_curr_addr, element_size, PostIndex));
214 __ Cmp(src_curr_addr, src_stop_addr);
215 __ B(&slow_copy_loop, ne);
216 __ B(GetExitLabel());
217 }
218
219 const char* GetDescription() const OVERRIDE { return "ReadBarrierSystemArrayCopySlowPathARM64"; }
220
221 private:
222 Location tmp_;
223
224 DISALLOW_COPY_AND_ASSIGN(ReadBarrierSystemArrayCopySlowPathARM64);
225};
Andreas Gampe878d58c2015-01-15 23:24:00 -0800226#undef __
227
228bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
229 Dispatch(invoke);
230 LocationSummary* res = invoke->GetLocations();
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000231 if (res == nullptr) {
232 return false;
233 }
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000234 return res->Intrinsified();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800235}
236
237#define __ masm->
238
Vladimir Markoca6fff82017-10-03 14:49:14 +0100239static void CreateFPToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) {
240 LocationSummary* locations =
241 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800242 locations->SetInAt(0, Location::RequiresFpuRegister());
243 locations->SetOut(Location::RequiresRegister());
244}
245
Vladimir Markoca6fff82017-10-03 14:49:14 +0100246static void CreateIntToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) {
247 LocationSummary* locations =
248 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800249 locations->SetInAt(0, Location::RequiresRegister());
250 locations->SetOut(Location::RequiresFpuRegister());
251}
252
Scott Wakeling97c72b72016-06-24 16:19:36 +0100253static void MoveFPToInt(LocationSummary* locations, bool is64bit, MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800254 Location input = locations->InAt(0);
255 Location output = locations->Out();
256 __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
257 is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
258}
259
Scott Wakeling97c72b72016-06-24 16:19:36 +0100260static void MoveIntToFP(LocationSummary* locations, bool is64bit, MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800261 Location input = locations->InAt(0);
262 Location output = locations->Out();
263 __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
264 is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
265}
266
267void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100268 CreateFPToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800269}
270void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100271 CreateIntToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800272}
273
274void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000275 MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800276}
277void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000278 MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800279}
280
281void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100282 CreateFPToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800283}
284void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100285 CreateIntToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800286}
287
288void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000289 MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800290}
291void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000292 MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800293}
294
Vladimir Markoca6fff82017-10-03 14:49:14 +0100295static void CreateIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) {
296 LocationSummary* locations =
297 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800298 locations->SetInAt(0, Location::RequiresRegister());
299 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
300}
301
302static void GenReverseBytes(LocationSummary* locations,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100303 DataType::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100304 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800305 Location in = locations->InAt(0);
306 Location out = locations->Out();
307
308 switch (type) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100309 case DataType::Type::kInt16:
Andreas Gampe878d58c2015-01-15 23:24:00 -0800310 __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
311 __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
312 break;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100313 case DataType::Type::kInt32:
314 case DataType::Type::kInt64:
Andreas Gampe878d58c2015-01-15 23:24:00 -0800315 __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
316 break;
317 default:
318 LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
319 UNREACHABLE();
320 }
321}
322
323void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100324 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800325}
326
327void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100328 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800329}
330
331void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100332 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800333}
334
335void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100336 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800337}
338
339void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100340 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800341}
342
343void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100344 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800345}
346
Vladimir Markoca6fff82017-10-03 14:49:14 +0100347static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) {
348 LocationSummary* locations =
349 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Aart Bik7b565022016-01-28 14:36:22 -0800350 locations->SetInAt(0, Location::RequiresRegister());
351 locations->SetInAt(1, Location::RequiresRegister());
352 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
353}
354
Scott Wakeling611d3392015-07-10 11:42:06 +0100355static void GenNumberOfLeadingZeros(LocationSummary* locations,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100356 DataType::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100357 MacroAssembler* masm) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100358 DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64);
Scott Wakeling611d3392015-07-10 11:42:06 +0100359
360 Location in = locations->InAt(0);
361 Location out = locations->Out();
362
363 __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
364}
365
366void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100367 CreateIntToIntLocations(allocator_, invoke);
Scott Wakeling611d3392015-07-10 11:42:06 +0100368}
369
370void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100371 GenNumberOfLeadingZeros(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler());
Scott Wakeling611d3392015-07-10 11:42:06 +0100372}
373
374void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100375 CreateIntToIntLocations(allocator_, invoke);
Scott Wakeling611d3392015-07-10 11:42:06 +0100376}
377
378void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100379 GenNumberOfLeadingZeros(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler());
Scott Wakeling611d3392015-07-10 11:42:06 +0100380}
381
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100382static void GenNumberOfTrailingZeros(LocationSummary* locations,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100383 DataType::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100384 MacroAssembler* masm) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100385 DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64);
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100386
387 Location in = locations->InAt(0);
388 Location out = locations->Out();
389
390 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
391 __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
392}
393
394void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100395 CreateIntToIntLocations(allocator_, invoke);
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100396}
397
398void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100399 GenNumberOfTrailingZeros(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler());
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100400}
401
402void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100403 CreateIntToIntLocations(allocator_, invoke);
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100404}
405
406void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100407 GenNumberOfTrailingZeros(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler());
Scott Wakeling9ee23f42015-07-23 10:44:35 +0100408}
409
Andreas Gampe878d58c2015-01-15 23:24:00 -0800410static void GenReverse(LocationSummary* locations,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100411 DataType::Type type,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100412 MacroAssembler* masm) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100413 DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800414
415 Location in = locations->InAt(0);
416 Location out = locations->Out();
417
418 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
419}
420
421void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100422 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800423}
424
425void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100426 GenReverse(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800427}
428
429void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100430 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800431}
432
433void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100434 GenReverse(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800435}
436
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100437static void GenBitCount(HInvoke* instr, DataType::Type type, MacroAssembler* masm) {
438 DCHECK(DataType::IsIntOrLongType(type)) << type;
439 DCHECK_EQ(instr->GetType(), DataType::Type::kInt32);
440 DCHECK_EQ(DataType::Kind(instr->InputAt(0)->GetType()), type);
xueliang.zhong49924c92016-03-03 10:52:51 +0000441
xueliang.zhong49924c92016-03-03 10:52:51 +0000442 UseScratchRegisterScope temps(masm);
443
Nicolas Geoffray457413a2016-03-04 11:10:17 +0000444 Register src = InputRegisterAt(instr, 0);
Roland Levillainfa3912e2016-04-01 18:21:55 +0100445 Register dst = RegisterFrom(instr->GetLocations()->Out(), type);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100446 FPRegister fpr = (type == DataType::Type::kInt64) ? temps.AcquireD() : temps.AcquireS();
xueliang.zhong49924c92016-03-03 10:52:51 +0000447
448 __ Fmov(fpr, src);
Nicolas Geoffray457413a2016-03-04 11:10:17 +0000449 __ Cnt(fpr.V8B(), fpr.V8B());
450 __ Addv(fpr.B(), fpr.V8B());
xueliang.zhong49924c92016-03-03 10:52:51 +0000451 __ Fmov(dst, fpr);
452}
453
454void IntrinsicLocationsBuilderARM64::VisitLongBitCount(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100455 CreateIntToIntLocations(allocator_, invoke);
xueliang.zhong49924c92016-03-03 10:52:51 +0000456}
457
458void IntrinsicCodeGeneratorARM64::VisitLongBitCount(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100459 GenBitCount(invoke, DataType::Type::kInt64, GetVIXLAssembler());
xueliang.zhong49924c92016-03-03 10:52:51 +0000460}
461
462void IntrinsicLocationsBuilderARM64::VisitIntegerBitCount(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100463 CreateIntToIntLocations(allocator_, invoke);
xueliang.zhong49924c92016-03-03 10:52:51 +0000464}
465
466void IntrinsicCodeGeneratorARM64::VisitIntegerBitCount(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100467 GenBitCount(invoke, DataType::Type::kInt32, GetVIXLAssembler());
xueliang.zhong49924c92016-03-03 10:52:51 +0000468}
469
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100470static void GenHighestOneBit(HInvoke* invoke, DataType::Type type, MacroAssembler* masm) {
471 DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64);
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100472
473 UseScratchRegisterScope temps(masm);
474
475 Register src = InputRegisterAt(invoke, 0);
476 Register dst = RegisterFrom(invoke->GetLocations()->Out(), type);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100477 Register temp = (type == DataType::Type::kInt64) ? temps.AcquireX() : temps.AcquireW();
478 size_t high_bit = (type == DataType::Type::kInt64) ? 63u : 31u;
479 size_t clz_high_bit = (type == DataType::Type::kInt64) ? 6u : 5u;
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100480
481 __ Clz(temp, src);
482 __ Mov(dst, UINT64_C(1) << high_bit); // MOV (bitmask immediate)
483 __ Bic(dst, dst, Operand(temp, LSL, high_bit - clz_high_bit)); // Clear dst if src was 0.
484 __ Lsr(dst, dst, temp);
485}
486
487void IntrinsicLocationsBuilderARM64::VisitIntegerHighestOneBit(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100488 CreateIntToIntLocations(allocator_, invoke);
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100489}
490
491void IntrinsicCodeGeneratorARM64::VisitIntegerHighestOneBit(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100492 GenHighestOneBit(invoke, DataType::Type::kInt32, GetVIXLAssembler());
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100493}
494
495void IntrinsicLocationsBuilderARM64::VisitLongHighestOneBit(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100496 CreateIntToIntLocations(allocator_, invoke);
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100497}
498
499void IntrinsicCodeGeneratorARM64::VisitLongHighestOneBit(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100500 GenHighestOneBit(invoke, DataType::Type::kInt64, GetVIXLAssembler());
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100501}
502
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100503static void GenLowestOneBit(HInvoke* invoke, DataType::Type type, MacroAssembler* masm) {
504 DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64);
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100505
506 UseScratchRegisterScope temps(masm);
507
508 Register src = InputRegisterAt(invoke, 0);
509 Register dst = RegisterFrom(invoke->GetLocations()->Out(), type);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100510 Register temp = (type == DataType::Type::kInt64) ? temps.AcquireX() : temps.AcquireW();
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100511
512 __ Neg(temp, src);
513 __ And(dst, temp, src);
514}
515
516void IntrinsicLocationsBuilderARM64::VisitIntegerLowestOneBit(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100517 CreateIntToIntLocations(allocator_, invoke);
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100518}
519
520void IntrinsicCodeGeneratorARM64::VisitIntegerLowestOneBit(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100521 GenLowestOneBit(invoke, DataType::Type::kInt32, GetVIXLAssembler());
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100522}
523
524void IntrinsicLocationsBuilderARM64::VisitLongLowestOneBit(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100525 CreateIntToIntLocations(allocator_, invoke);
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100526}
527
528void IntrinsicCodeGeneratorARM64::VisitLongLowestOneBit(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100529 GenLowestOneBit(invoke, DataType::Type::kInt64, GetVIXLAssembler());
Petre-Ionut Tudorda483162017-08-14 13:54:31 +0100530}
531
Vladimir Markoca6fff82017-10-03 14:49:14 +0100532static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) {
533 LocationSummary* locations =
534 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800535 locations->SetInAt(0, Location::RequiresFpuRegister());
536 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
537}
538
Scott Wakeling97c72b72016-06-24 16:19:36 +0100539static void MathAbsFP(LocationSummary* locations, bool is64bit, MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800540 Location in = locations->InAt(0);
541 Location out = locations->Out();
542
543 FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
544 FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
545
546 __ Fabs(out_reg, in_reg);
547}
548
549void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100550 CreateFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800551}
552
553void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000554 MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800555}
556
557void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100558 CreateFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800559}
560
561void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000562 MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800563}
564
Andreas Gampe878d58c2015-01-15 23:24:00 -0800565static void GenAbsInteger(LocationSummary* locations,
566 bool is64bit,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100567 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800568 Location in = locations->InAt(0);
569 Location output = locations->Out();
570
571 Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
572 Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
573
574 __ Cmp(in_reg, Operand(0));
575 __ Cneg(out_reg, in_reg, lt);
576}
577
578void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100579 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800580}
581
582void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000583 GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800584}
585
586void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100587 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800588}
589
590void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000591 GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800592}
593
594static void GenMinMaxFP(LocationSummary* locations,
595 bool is_min,
596 bool is_double,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100597 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800598 Location op1 = locations->InAt(0);
599 Location op2 = locations->InAt(1);
600 Location out = locations->Out();
601
602 FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
603 FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
604 FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
605 if (is_min) {
606 __ Fmin(out_reg, op1_reg, op2_reg);
607 } else {
608 __ Fmax(out_reg, op1_reg, op2_reg);
609 }
610}
611
Vladimir Markoca6fff82017-10-03 14:49:14 +0100612static void CreateFPFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) {
613 LocationSummary* locations =
614 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800615 locations->SetInAt(0, Location::RequiresFpuRegister());
616 locations->SetInAt(1, Location::RequiresFpuRegister());
617 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
618}
619
620void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100621 CreateFPFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800622}
623
624void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000625 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800626}
627
628void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100629 CreateFPFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800630}
631
632void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000633 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800634}
635
636void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100637 CreateFPFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800638}
639
640void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000641 GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800642}
643
644void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100645 CreateFPFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800646}
647
648void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000649 GenMinMaxFP(
650 invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800651}
652
653static void GenMinMax(LocationSummary* locations,
654 bool is_min,
655 bool is_long,
Scott Wakeling97c72b72016-06-24 16:19:36 +0100656 MacroAssembler* masm) {
Andreas Gampe878d58c2015-01-15 23:24:00 -0800657 Location op1 = locations->InAt(0);
658 Location op2 = locations->InAt(1);
659 Location out = locations->Out();
660
661 Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
662 Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
663 Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
664
665 __ Cmp(op1_reg, op2_reg);
666 __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
667}
668
Andreas Gampe878d58c2015-01-15 23:24:00 -0800669void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100670 CreateIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800671}
672
673void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000674 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800675}
676
677void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100678 CreateIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800679}
680
681void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000682 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800683}
684
685void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100686 CreateIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800687}
688
689void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000690 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800691}
692
693void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100694 CreateIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800695}
696
697void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +0000698 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800699}
700
701void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100702 CreateFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800703}
704
705void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
706 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100707 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800708 __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
709}
710
711void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100712 CreateFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800713}
714
715void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
716 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100717 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800718 __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
719}
720
721void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100722 CreateFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800723}
724
725void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
726 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100727 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800728 __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
729}
730
731void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100732 CreateFPToFPLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800733}
734
735void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
736 LocationSummary* locations = invoke->GetLocations();
Scott Wakeling97c72b72016-06-24 16:19:36 +0100737 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800738 __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
739}
740
Vladimir Markoca6fff82017-10-03 14:49:14 +0100741static void CreateFPToIntPlusFPTempLocations(ArenaAllocator* allocator, HInvoke* invoke) {
742 LocationSummary* locations =
743 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800744 locations->SetInAt(0, Location::RequiresFpuRegister());
745 locations->SetOut(Location::RequiresRegister());
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100746 locations->AddTemp(Location::RequiresFpuRegister());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800747}
748
Scott Wakeling97c72b72016-06-24 16:19:36 +0100749static void GenMathRound(HInvoke* invoke, bool is_double, vixl::aarch64::MacroAssembler* masm) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100750 // Java 8 API definition for Math.round():
751 // Return the closest long or int to the argument, with ties rounding to positive infinity.
752 //
753 // There is no single instruction in ARMv8 that can support the above definition.
754 // We choose to use FCVTAS here, because it has closest semantic.
755 // FCVTAS performs rounding to nearest integer, ties away from zero.
756 // For most inputs (positive values, zero or NaN), this instruction is enough.
757 // We only need a few handling code after FCVTAS if the input is negative half value.
758 //
759 // The reason why we didn't choose FCVTPS instruction here is that
760 // although it performs rounding toward positive infinity, it doesn't perform rounding to nearest.
761 // For example, FCVTPS(-1.9) = -1 and FCVTPS(1.1) = 2.
762 // If we were using this instruction, for most inputs, more handling code would be needed.
763 LocationSummary* l = invoke->GetLocations();
764 FPRegister in_reg = is_double ? DRegisterFrom(l->InAt(0)) : SRegisterFrom(l->InAt(0));
765 FPRegister tmp_fp = is_double ? DRegisterFrom(l->GetTemp(0)) : SRegisterFrom(l->GetTemp(0));
766 Register out_reg = is_double ? XRegisterFrom(l->Out()) : WRegisterFrom(l->Out());
Scott Wakeling97c72b72016-06-24 16:19:36 +0100767 vixl::aarch64::Label done;
Andreas Gampe878d58c2015-01-15 23:24:00 -0800768
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100769 // Round to nearest integer, ties away from zero.
770 __ Fcvtas(out_reg, in_reg);
771
772 // For positive values, zero or NaN inputs, rounding is done.
Scott Wakeling97c72b72016-06-24 16:19:36 +0100773 __ Tbz(out_reg, out_reg.GetSizeInBits() - 1, &done);
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100774
775 // Handle input < 0 cases.
776 // If input is negative but not a tie, previous result (round to nearest) is valid.
777 // If input is a negative tie, out_reg += 1.
778 __ Frinta(tmp_fp, in_reg);
779 __ Fsub(tmp_fp, in_reg, tmp_fp);
780 __ Fcmp(tmp_fp, 0.5);
781 __ Cinc(out_reg, out_reg, eq);
782
783 __ Bind(&done);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800784}
785
786void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100787 CreateFPToIntPlusFPTempLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800788}
789
790void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100791 GenMathRound(invoke, /* is_double */ true, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800792}
793
794void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100795 CreateFPToIntPlusFPTempLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800796}
797
798void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
xueliang.zhongd1e153c2016-05-27 18:56:13 +0100799 GenMathRound(invoke, /* is_double */ false, GetVIXLAssembler());
Andreas Gampe878d58c2015-01-15 23:24:00 -0800800}
801
802void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100803 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800804}
805
806void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100807 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800808 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
809 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
810}
811
812void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100813 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800814}
815
816void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100817 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800818 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
819 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
820}
821
822void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100823 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800824}
825
826void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100827 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800828 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
829 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
830}
831
832void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100833 CreateIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800834}
835
836void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100837 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800838 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
839 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
840}
841
Vladimir Markoca6fff82017-10-03 14:49:14 +0100842static void CreateIntIntToVoidLocations(ArenaAllocator* allocator, HInvoke* invoke) {
843 LocationSummary* locations =
844 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800845 locations->SetInAt(0, Location::RequiresRegister());
846 locations->SetInAt(1, Location::RequiresRegister());
847}
848
849void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100850 CreateIntIntToVoidLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800851}
852
853void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100854 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800855 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
856 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
857}
858
859void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100860 CreateIntIntToVoidLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800861}
862
863void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100864 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800865 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
866 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
867}
868
869void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100870 CreateIntIntToVoidLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800871}
872
873void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100874 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800875 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
876 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
877}
878
879void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100880 CreateIntIntToVoidLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800881}
882
883void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +0100884 MacroAssembler* masm = GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -0800885 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
886 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
887}
888
889void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100890 LocationSummary* locations =
891 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800892 locations->SetOut(Location::RequiresRegister());
893}
894
895void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100896 codegen_->Load(DataType::Type::kReference, WRegisterFrom(invoke->GetLocations()->Out()),
Andreas Gampe542451c2016-07-26 09:02:02 -0700897 MemOperand(tr, Thread::PeerOffset<kArm64PointerSize>().Int32Value()));
Andreas Gampe878d58c2015-01-15 23:24:00 -0800898}
899
900static void GenUnsafeGet(HInvoke* invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100901 DataType::Type type,
Andreas Gampe878d58c2015-01-15 23:24:00 -0800902 bool is_volatile,
903 CodeGeneratorARM64* codegen) {
904 LocationSummary* locations = invoke->GetLocations();
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100905 DCHECK((type == DataType::Type::kInt32) ||
906 (type == DataType::Type::kInt64) ||
907 (type == DataType::Type::kReference));
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000908 Location base_loc = locations->InAt(1);
909 Register base = WRegisterFrom(base_loc); // Object pointer.
910 Location offset_loc = locations->InAt(2);
911 Register offset = XRegisterFrom(offset_loc); // Long offset.
912 Location trg_loc = locations->Out();
913 Register trg = RegisterFrom(trg_loc, type);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800914
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100915 if (type == DataType::Type::kReference && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
Roland Levillain44015862016-01-22 11:47:17 +0000916 // UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case.
Roland Levillain54f869e2017-03-06 13:54:11 +0000917 Register temp = WRegisterFrom(locations->GetTemp(0));
Roland Levillainbfea3352016-06-23 13:48:47 +0100918 codegen->GenerateReferenceLoadWithBakerReadBarrier(invoke,
919 trg_loc,
920 base,
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100921 /* offset */ 0u,
Roland Levillainbfea3352016-06-23 13:48:47 +0100922 /* index */ offset_loc,
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100923 /* scale_factor */ 0u,
Roland Levillainbfea3352016-06-23 13:48:47 +0100924 temp,
925 /* needs_null_check */ false,
926 is_volatile);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800927 } else {
Roland Levillain44015862016-01-22 11:47:17 +0000928 // Other cases.
929 MemOperand mem_op(base.X(), offset);
930 if (is_volatile) {
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +0000931 codegen->LoadAcquire(invoke, trg, mem_op, /* needs_null_check */ true);
Roland Levillain44015862016-01-22 11:47:17 +0000932 } else {
933 codegen->Load(type, trg, mem_op);
934 }
Roland Levillain4d027112015-07-01 15:41:14 +0100935
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100936 if (type == DataType::Type::kReference) {
Roland Levillain44015862016-01-22 11:47:17 +0000937 DCHECK(trg.IsW());
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100938 codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0u, offset_loc);
Roland Levillain44015862016-01-22 11:47:17 +0000939 }
Roland Levillain4d027112015-07-01 15:41:14 +0100940 }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800941}
942
Vladimir Markoca6fff82017-10-03 14:49:14 +0100943static void CreateIntIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) {
Roland Levillain22ccc3a2015-11-24 13:10:05 +0000944 bool can_call = kEmitCompilerReadBarrier &&
945 (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
946 invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
Vladimir Markoca6fff82017-10-03 14:49:14 +0100947 LocationSummary* locations =
948 new (allocator) LocationSummary(invoke,
949 can_call
950 ? LocationSummary::kCallOnSlowPath
951 : LocationSummary::kNoCall,
952 kIntrinsified);
Vladimir Marko70e97462016-08-09 11:04:26 +0100953 if (can_call && kUseBakerReadBarrier) {
Vladimir Marko804b03f2016-09-14 16:26:36 +0100954 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
Roland Levillain54f869e2017-03-06 13:54:11 +0000955 // We need a temporary register for the read barrier marking slow
956 // path in CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier.
957 locations->AddTemp(Location::RequiresRegister());
Vladimir Marko70e97462016-08-09 11:04:26 +0100958 }
Andreas Gampe878d58c2015-01-15 23:24:00 -0800959 locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
960 locations->SetInAt(1, Location::RequiresRegister());
961 locations->SetInAt(2, Location::RequiresRegister());
Roland Levillainbfea3352016-06-23 13:48:47 +0100962 locations->SetOut(Location::RequiresRegister(),
Roland Levillaina1aa3b12016-10-26 13:03:38 +0100963 (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap));
Andreas Gampe878d58c2015-01-15 23:24:00 -0800964}
965
966void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100967 CreateIntIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800968}
969void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100970 CreateIntIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800971}
972void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100973 CreateIntIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800974}
975void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100976 CreateIntIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800977}
978void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100979 CreateIntIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800980}
981void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +0100982 CreateIntIntIntToIntLocations(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800983}
984
985void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100986 GenUnsafeGet(invoke, DataType::Type::kInt32, /* is_volatile */ false, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800987}
988void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100989 GenUnsafeGet(invoke, DataType::Type::kInt32, /* is_volatile */ true, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800990}
991void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100992 GenUnsafeGet(invoke, DataType::Type::kInt64, /* is_volatile */ false, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800993}
994void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100995 GenUnsafeGet(invoke, DataType::Type::kInt64, /* is_volatile */ true, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800996}
997void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +0100998 GenUnsafeGet(invoke, DataType::Type::kReference, /* is_volatile */ false, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -0800999}
1000void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001001 GenUnsafeGet(invoke, DataType::Type::kReference, /* is_volatile */ true, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001002}
1003
Vladimir Markoca6fff82017-10-03 14:49:14 +01001004static void CreateIntIntIntIntToVoid(ArenaAllocator* allocator, HInvoke* invoke) {
1005 LocationSummary* locations =
1006 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001007 locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
1008 locations->SetInAt(1, Location::RequiresRegister());
1009 locations->SetInAt(2, Location::RequiresRegister());
1010 locations->SetInAt(3, Location::RequiresRegister());
1011}
1012
1013void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001014 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001015}
1016void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001017 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001018}
1019void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001020 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001021}
1022void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001023 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001024}
1025void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001026 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001027}
1028void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001029 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001030}
1031void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001032 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001033}
1034void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001035 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001036}
1037void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001038 CreateIntIntIntIntToVoid(allocator_, invoke);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001039}
1040
Artem Serov914d7a82017-02-07 14:33:49 +00001041static void GenUnsafePut(HInvoke* invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001042 DataType::Type type,
Andreas Gampe878d58c2015-01-15 23:24:00 -08001043 bool is_volatile,
1044 bool is_ordered,
1045 CodeGeneratorARM64* codegen) {
Artem Serov914d7a82017-02-07 14:33:49 +00001046 LocationSummary* locations = invoke->GetLocations();
Alexandre Rames087930f2016-08-02 13:45:28 +01001047 MacroAssembler* masm = codegen->GetVIXLAssembler();
Andreas Gampe878d58c2015-01-15 23:24:00 -08001048
1049 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer.
1050 Register offset = XRegisterFrom(locations->InAt(2)); // Long offset.
1051 Register value = RegisterFrom(locations->InAt(3), type);
Roland Levillain4d027112015-07-01 15:41:14 +01001052 Register source = value;
Andreas Gampe878d58c2015-01-15 23:24:00 -08001053 MemOperand mem_op(base.X(), offset);
1054
Roland Levillain4d027112015-07-01 15:41:14 +01001055 {
1056 // We use a block to end the scratch scope before the write barrier, thus
1057 // freeing the temporary registers so they can be used in `MarkGCCard`.
1058 UseScratchRegisterScope temps(masm);
1059
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001060 if (kPoisonHeapReferences && type == DataType::Type::kReference) {
Roland Levillain4d027112015-07-01 15:41:14 +01001061 DCHECK(value.IsW());
1062 Register temp = temps.AcquireW();
1063 __ Mov(temp.W(), value.W());
1064 codegen->GetAssembler()->PoisonHeapReference(temp.W());
1065 source = temp;
Andreas Gampe878d58c2015-01-15 23:24:00 -08001066 }
Roland Levillain4d027112015-07-01 15:41:14 +01001067
1068 if (is_volatile || is_ordered) {
Artem Serov914d7a82017-02-07 14:33:49 +00001069 codegen->StoreRelease(invoke, type, source, mem_op, /* needs_null_check */ false);
Roland Levillain4d027112015-07-01 15:41:14 +01001070 } else {
1071 codegen->Store(type, source, mem_op);
1072 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001073 }
1074
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001075 if (type == DataType::Type::kReference) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001076 bool value_can_be_null = true; // TODO: Worth finding out this information?
1077 codegen->MarkGCCard(base, value, value_can_be_null);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001078 }
1079}
1080
1081void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001082 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001083 DataType::Type::kInt32,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001084 /* is_volatile */ false,
1085 /* is_ordered */ false,
1086 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001087}
1088void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001089 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001090 DataType::Type::kInt32,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001091 /* is_volatile */ false,
1092 /* is_ordered */ true,
1093 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001094}
1095void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001096 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001097 DataType::Type::kInt32,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001098 /* is_volatile */ true,
1099 /* is_ordered */ false,
1100 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001101}
1102void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001103 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001104 DataType::Type::kReference,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001105 /* is_volatile */ false,
1106 /* is_ordered */ false,
1107 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001108}
1109void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001110 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001111 DataType::Type::kReference,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001112 /* is_volatile */ false,
1113 /* is_ordered */ true,
1114 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001115}
1116void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001117 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001118 DataType::Type::kReference,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001119 /* is_volatile */ true,
1120 /* is_ordered */ false,
1121 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001122}
1123void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001124 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001125 DataType::Type::kInt64,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001126 /* is_volatile */ false,
1127 /* is_ordered */ false,
1128 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001129}
1130void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001131 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001132 DataType::Type::kInt64,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001133 /* is_volatile */ false,
1134 /* is_ordered */ true,
1135 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001136}
1137void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
Artem Serov914d7a82017-02-07 14:33:49 +00001138 GenUnsafePut(invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001139 DataType::Type::kInt64,
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001140 /* is_volatile */ true,
1141 /* is_ordered */ false,
1142 codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001143}
1144
Vladimir Markoca6fff82017-10-03 14:49:14 +01001145static void CreateIntIntIntIntIntToInt(ArenaAllocator* allocator,
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001146 HInvoke* invoke,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001147 DataType::Type type) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001148 bool can_call = kEmitCompilerReadBarrier &&
1149 kUseBakerReadBarrier &&
1150 (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject);
Vladimir Markoca6fff82017-10-03 14:49:14 +01001151 LocationSummary* locations =
1152 new (allocator) LocationSummary(invoke,
1153 can_call
1154 ? LocationSummary::kCallOnSlowPath
1155 : LocationSummary::kNoCall,
1156 kIntrinsified);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001157 locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
1158 locations->SetInAt(1, Location::RequiresRegister());
1159 locations->SetInAt(2, Location::RequiresRegister());
1160 locations->SetInAt(3, Location::RequiresRegister());
1161 locations->SetInAt(4, Location::RequiresRegister());
1162
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001163 // If heap poisoning is enabled, we don't want the unpoisoning
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001164 // operations to potentially clobber the output. Likewise when
1165 // emitting a (Baker) read barrier, which may call.
1166 Location::OutputOverlap overlaps =
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001167 ((kPoisonHeapReferences && type == DataType::Type::kReference) || can_call)
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001168 ? Location::kOutputOverlap
1169 : Location::kNoOutputOverlap;
1170 locations->SetOut(Location::RequiresRegister(), overlaps);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001171 if (type == DataType::Type::kReference && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001172 // Temporary register for (Baker) read barrier.
1173 locations->AddTemp(Location::RequiresRegister());
1174 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001175}
1176
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001177static void GenCas(HInvoke* invoke, DataType::Type type, CodeGeneratorARM64* codegen) {
Alexandre Rames087930f2016-08-02 13:45:28 +01001178 MacroAssembler* masm = codegen->GetVIXLAssembler();
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001179 LocationSummary* locations = invoke->GetLocations();
Andreas Gampe878d58c2015-01-15 23:24:00 -08001180
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001181 Location out_loc = locations->Out();
1182 Register out = WRegisterFrom(out_loc); // Boolean result.
Andreas Gampe878d58c2015-01-15 23:24:00 -08001183
1184 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer.
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001185 Location offset_loc = locations->InAt(2);
1186 Register offset = XRegisterFrom(offset_loc); // Long offset.
Andreas Gampe878d58c2015-01-15 23:24:00 -08001187 Register expected = RegisterFrom(locations->InAt(3), type); // Expected.
1188 Register value = RegisterFrom(locations->InAt(4), type); // Value.
1189
1190 // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001191 if (type == DataType::Type::kReference) {
Andreas Gampe878d58c2015-01-15 23:24:00 -08001192 // Mark card for object assuming new value is stored.
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001193 bool value_can_be_null = true; // TODO: Worth finding out this information?
1194 codegen->MarkGCCard(base, value, value_can_be_null);
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001195
1196 // The only read barrier implementation supporting the
1197 // UnsafeCASObject intrinsic is the Baker-style read barriers.
1198 DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier);
1199
1200 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
1201 Register temp = WRegisterFrom(locations->GetTemp(0));
1202 // Need to make sure the reference stored in the field is a to-space
1203 // one before attempting the CAS or the CAS could fail incorrectly.
Roland Levillainff487002017-03-07 16:50:01 +00001204 codegen->UpdateReferenceFieldWithBakerReadBarrier(
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001205 invoke,
1206 out_loc, // Unused, used only as a "temporary" within the read barrier.
1207 base,
Roland Levillainff487002017-03-07 16:50:01 +00001208 /* field_offset */ offset_loc,
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001209 temp,
1210 /* needs_null_check */ false,
Roland Levillainff487002017-03-07 16:50:01 +00001211 /* use_load_acquire */ false);
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001212 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001213 }
1214
1215 UseScratchRegisterScope temps(masm);
1216 Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory.
1217 Register tmp_value = temps.AcquireSameSizeAs(value); // Value in memory.
1218
1219 Register tmp_32 = tmp_value.W();
1220
1221 __ Add(tmp_ptr, base.X(), Operand(offset));
1222
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001223 if (kPoisonHeapReferences && type == DataType::Type::kReference) {
Roland Levillain4d027112015-07-01 15:41:14 +01001224 codegen->GetAssembler()->PoisonHeapReference(expected);
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001225 if (value.Is(expected)) {
1226 // Do not poison `value`, as it is the same register as
1227 // `expected`, which has just been poisoned.
1228 } else {
1229 codegen->GetAssembler()->PoisonHeapReference(value);
1230 }
Roland Levillain4d027112015-07-01 15:41:14 +01001231 }
1232
Andreas Gampe878d58c2015-01-15 23:24:00 -08001233 // do {
1234 // tmp_value = [tmp_ptr] - expected;
1235 // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1236 // result = tmp_value != 0;
1237
Scott Wakeling97c72b72016-06-24 16:19:36 +01001238 vixl::aarch64::Label loop_head, exit_loop;
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001239 __ Bind(&loop_head);
Serban Constantinescu4a6a67c2016-01-27 09:19:56 +00001240 __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1241 __ Cmp(tmp_value, expected);
1242 __ B(&exit_loop, ne);
1243 __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1244 __ Cbnz(tmp_32, &loop_head);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001245 __ Bind(&exit_loop);
1246 __ Cset(out, eq);
Roland Levillain4d027112015-07-01 15:41:14 +01001247
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001248 if (kPoisonHeapReferences && type == DataType::Type::kReference) {
Roland Levillain4d027112015-07-01 15:41:14 +01001249 codegen->GetAssembler()->UnpoisonHeapReference(expected);
Roland Levillain2e50ecb2016-01-27 14:08:33 +00001250 if (value.Is(expected)) {
1251 // Do not unpoison `value`, as it is the same register as
1252 // `expected`, which has just been unpoisoned.
1253 } else {
1254 codegen->GetAssembler()->UnpoisonHeapReference(value);
1255 }
Roland Levillain4d027112015-07-01 15:41:14 +01001256 }
Andreas Gampe878d58c2015-01-15 23:24:00 -08001257}
1258
1259void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001260 CreateIntIntIntIntIntToInt(allocator_, invoke, DataType::Type::kInt32);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001261}
1262void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001263 CreateIntIntIntIntIntToInt(allocator_, invoke, DataType::Type::kInt64);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001264}
1265void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001266 // The only read barrier implementation supporting the
1267 // UnsafeCASObject intrinsic is the Baker-style read barriers.
1268 if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) {
Roland Levillain985ff702015-10-23 13:25:35 +01001269 return;
1270 }
1271
Vladimir Markoca6fff82017-10-03 14:49:14 +01001272 CreateIntIntIntIntIntToInt(allocator_, invoke, DataType::Type::kReference);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001273}
1274
1275void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001276 GenCas(invoke, DataType::Type::kInt32, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001277}
1278void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001279 GenCas(invoke, DataType::Type::kInt64, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001280}
1281void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
Roland Levillaina1aa3b12016-10-26 13:03:38 +01001282 // The only read barrier implementation supporting the
1283 // UnsafeCASObject intrinsic is the Baker-style read barriers.
1284 DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier);
Roland Levillain3d312422016-06-23 13:53:42 +01001285
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001286 GenCas(invoke, DataType::Type::kReference, codegen_);
Andreas Gampe878d58c2015-01-15 23:24:00 -08001287}
1288
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001289void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001290 LocationSummary* locations =
1291 new (allocator_) LocationSummary(invoke,
1292 invoke->InputAt(1)->CanBeNull()
1293 ? LocationSummary::kCallOnSlowPath
1294 : LocationSummary::kNoCall,
1295 kIntrinsified);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001296 locations->SetInAt(0, Location::RequiresRegister());
1297 locations->SetInAt(1, Location::RequiresRegister());
1298 locations->AddTemp(Location::RequiresRegister());
1299 locations->AddTemp(Location::RequiresRegister());
1300 locations->AddTemp(Location::RequiresRegister());
jessicahandojo05765752016-09-09 19:01:32 -07001301 // Need temporary registers for String compression's feature.
1302 if (mirror::kUseStringCompression) {
1303 locations->AddTemp(Location::RequiresRegister());
jessicahandojo05765752016-09-09 19:01:32 -07001304 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001305 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001306}
1307
1308void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001309 MacroAssembler* masm = GetVIXLAssembler();
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001310 LocationSummary* locations = invoke->GetLocations();
1311
Alexandre Rames2ea91532016-08-11 17:04:14 +01001312 Register str = InputRegisterAt(invoke, 0);
1313 Register arg = InputRegisterAt(invoke, 1);
1314 DCHECK(str.IsW());
1315 DCHECK(arg.IsW());
Scott Wakeling1f36f412016-04-21 11:13:45 +01001316 Register out = OutputRegister(invoke);
1317
1318 Register temp0 = WRegisterFrom(locations->GetTemp(0));
1319 Register temp1 = WRegisterFrom(locations->GetTemp(1));
1320 Register temp2 = WRegisterFrom(locations->GetTemp(2));
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001321 Register temp3;
jessicahandojo05765752016-09-09 19:01:32 -07001322 if (mirror::kUseStringCompression) {
1323 temp3 = WRegisterFrom(locations->GetTemp(3));
jessicahandojo05765752016-09-09 19:01:32 -07001324 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001325
Scott Wakeling97c72b72016-06-24 16:19:36 +01001326 vixl::aarch64::Label loop;
1327 vixl::aarch64::Label find_char_diff;
1328 vixl::aarch64::Label end;
jessicahandojo05765752016-09-09 19:01:32 -07001329 vixl::aarch64::Label different_compression;
Scott Wakeling1f36f412016-04-21 11:13:45 +01001330
1331 // Get offsets of count and value fields within a string object.
1332 const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1333 const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1334
Nicolas Geoffray512e04d2015-03-27 17:21:24 +00001335 // Note that the null check must have been done earlier.
Calin Juravle641547a2015-04-21 22:08:51 +01001336 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001337
Scott Wakeling1f36f412016-04-21 11:13:45 +01001338 // Take slow path and throw if input can be and is null.
1339 SlowPathCodeARM64* slow_path = nullptr;
1340 const bool can_slow_path = invoke->InputAt(1)->CanBeNull();
1341 if (can_slow_path) {
1342 slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1343 codegen_->AddSlowPath(slow_path);
1344 __ Cbz(arg, slow_path->GetEntryLabel());
1345 }
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001346
Scott Wakeling1f36f412016-04-21 11:13:45 +01001347 // Reference equality check, return 0 if same reference.
1348 __ Subs(out, str, arg);
1349 __ B(&end, eq);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001350
jessicahandojo05765752016-09-09 19:01:32 -07001351 if (mirror::kUseStringCompression) {
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001352 // Load `count` fields of this and argument strings.
jessicahandojo05765752016-09-09 19:01:32 -07001353 __ Ldr(temp3, HeapOperand(str, count_offset));
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001354 __ Ldr(temp2, HeapOperand(arg, count_offset));
jessicahandojo05765752016-09-09 19:01:32 -07001355 // Clean out compression flag from lengths.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001356 __ Lsr(temp0, temp3, 1u);
1357 __ Lsr(temp1, temp2, 1u);
jessicahandojo05765752016-09-09 19:01:32 -07001358 } else {
1359 // Load lengths of this and argument strings.
1360 __ Ldr(temp0, HeapOperand(str, count_offset));
1361 __ Ldr(temp1, HeapOperand(arg, count_offset));
1362 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001363 // out = length diff.
1364 __ Subs(out, temp0, temp1);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001365 // temp0 = min(len(str), len(arg)).
1366 __ Csel(temp0, temp1, temp0, ge);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001367 // Shorter string is empty?
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001368 __ Cbz(temp0, &end);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001369
jessicahandojo05765752016-09-09 19:01:32 -07001370 if (mirror::kUseStringCompression) {
1371 // Check if both strings using same compression style to use this comparison loop.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001372 __ Eor(temp2, temp2, Operand(temp3));
1373 // Interleave with compression flag extraction which is needed for both paths
1374 // and also set flags which is needed only for the different compressions path.
1375 __ Ands(temp3.W(), temp3.W(), Operand(1));
1376 __ Tbnz(temp2, 0, &different_compression); // Does not use flags.
jessicahandojo05765752016-09-09 19:01:32 -07001377 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001378 // Store offset of string value in preparation for comparison loop.
1379 __ Mov(temp1, value_offset);
jessicahandojo05765752016-09-09 19:01:32 -07001380 if (mirror::kUseStringCompression) {
1381 // For string compression, calculate the number of bytes to compare (not chars).
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001382 // This could in theory exceed INT32_MAX, so treat temp0 as unsigned.
1383 __ Lsl(temp0, temp0, temp3);
jessicahandojo05765752016-09-09 19:01:32 -07001384 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001385
1386 UseScratchRegisterScope scratch_scope(masm);
1387 Register temp4 = scratch_scope.AcquireX();
1388
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001389 // Assertions that must hold in order to compare strings 8 bytes at a time.
Scott Wakeling1f36f412016-04-21 11:13:45 +01001390 DCHECK_ALIGNED(value_offset, 8);
1391 static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1392
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001393 const size_t char_size = DataType::Size(DataType::Type::kUint16);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001394 DCHECK_EQ(char_size, 2u);
1395
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001396 // Promote temp2 to an X reg, ready for LDR.
1397 temp2 = temp2.X();
Scott Wakeling1f36f412016-04-21 11:13:45 +01001398
1399 // Loop to compare 4x16-bit characters at a time (ok because of string data alignment).
1400 __ Bind(&loop);
Alexandre Rames2ea91532016-08-11 17:04:14 +01001401 __ Ldr(temp4, MemOperand(str.X(), temp1.X()));
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001402 __ Ldr(temp2, MemOperand(arg.X(), temp1.X()));
1403 __ Cmp(temp4, temp2);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001404 __ B(ne, &find_char_diff);
1405 __ Add(temp1, temp1, char_size * 4);
jessicahandojo05765752016-09-09 19:01:32 -07001406 // With string compression, we have compared 8 bytes, otherwise 4 chars.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001407 __ Subs(temp0, temp0, (mirror::kUseStringCompression) ? 8 : 4);
1408 __ B(&loop, hi);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001409 __ B(&end);
1410
1411 // Promote temp1 to an X reg, ready for EOR.
1412 temp1 = temp1.X();
1413
jessicahandojo05765752016-09-09 19:01:32 -07001414 // Find the single character difference.
Scott Wakeling1f36f412016-04-21 11:13:45 +01001415 __ Bind(&find_char_diff);
1416 // Get the bit position of the first character that differs.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001417 __ Eor(temp1, temp2, temp4);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001418 __ Rbit(temp1, temp1);
1419 __ Clz(temp1, temp1);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001420
jessicahandojo05765752016-09-09 19:01:32 -07001421 // If the number of chars remaining <= the index where the difference occurs (0-3), then
Scott Wakeling1f36f412016-04-21 11:13:45 +01001422 // the difference occurs outside the remaining string data, so just return length diff (out).
jessicahandojo05765752016-09-09 19:01:32 -07001423 // Unlike ARM, we're doing the comparison in one go here, without the subtraction at the
1424 // find_char_diff_2nd_cmp path, so it doesn't matter whether the comparison is signed or
1425 // unsigned when string compression is disabled.
1426 // When it's enabled, the comparison must be unsigned.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001427 __ Cmp(temp0, Operand(temp1.W(), LSR, (mirror::kUseStringCompression) ? 3 : 4));
jessicahandojo05765752016-09-09 19:01:32 -07001428 __ B(ls, &end);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001429
Scott Wakeling1f36f412016-04-21 11:13:45 +01001430 // Extract the characters and calculate the difference.
jessicahandojo05765752016-09-09 19:01:32 -07001431 if (mirror:: kUseStringCompression) {
jessicahandojo05765752016-09-09 19:01:32 -07001432 __ Bic(temp1, temp1, 0x7);
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001433 __ Bic(temp1, temp1, Operand(temp3.X(), LSL, 3u));
1434 } else {
1435 __ Bic(temp1, temp1, 0xf);
jessicahandojo05765752016-09-09 19:01:32 -07001436 }
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001437 __ Lsr(temp2, temp2, temp1);
Scott Wakeling1f36f412016-04-21 11:13:45 +01001438 __ Lsr(temp4, temp4, temp1);
jessicahandojo05765752016-09-09 19:01:32 -07001439 if (mirror::kUseStringCompression) {
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001440 // Prioritize the case of compressed strings and calculate such result first.
1441 __ Uxtb(temp1, temp4);
1442 __ Sub(out, temp1.W(), Operand(temp2.W(), UXTB));
1443 __ Tbz(temp3, 0u, &end); // If actually compressed, we're done.
jessicahandojo05765752016-09-09 19:01:32 -07001444 }
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001445 __ Uxth(temp4, temp4);
1446 __ Sub(out, temp4.W(), Operand(temp2.W(), UXTH));
jessicahandojo05765752016-09-09 19:01:32 -07001447
1448 if (mirror::kUseStringCompression) {
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001449 __ B(&end);
1450 __ Bind(&different_compression);
1451
1452 // Comparison for different compression style.
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001453 const size_t c_char_size = DataType::Size(DataType::Type::kInt8);
jessicahandojo05765752016-09-09 19:01:32 -07001454 DCHECK_EQ(c_char_size, 1u);
jessicahandojo05765752016-09-09 19:01:32 -07001455 temp1 = temp1.W();
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001456 temp2 = temp2.W();
1457 temp4 = temp4.W();
jessicahandojo05765752016-09-09 19:01:32 -07001458
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001459 // `temp1` will hold the compressed data pointer, `temp2` the uncompressed data pointer.
1460 // Note that flags have been set by the `str` compression flag extraction to `temp3`
1461 // before branching to the `different_compression` label.
1462 __ Csel(temp1, str, arg, eq); // Pointer to the compressed string.
1463 __ Csel(temp2, str, arg, ne); // Pointer to the uncompressed string.
jessicahandojo05765752016-09-09 19:01:32 -07001464
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001465 // We want to free up the temp3, currently holding `str` compression flag, for comparison.
1466 // So, we move it to the bottom bit of the iteration count `temp0` which we then need to treat
1467 // as unsigned. Start by freeing the bit with a LSL and continue further down by a SUB which
1468 // will allow `subs temp0, #2; bhi different_compression_loop` to serve as the loop condition.
1469 __ Lsl(temp0, temp0, 1u);
1470
1471 // Adjust temp1 and temp2 from string pointers to data pointers.
1472 __ Add(temp1, temp1, Operand(value_offset));
1473 __ Add(temp2, temp2, Operand(value_offset));
1474
1475 // Complete the move of the compression flag.
1476 __ Sub(temp0, temp0, Operand(temp3));
1477
1478 vixl::aarch64::Label different_compression_loop;
1479 vixl::aarch64::Label different_compression_diff;
1480
1481 __ Bind(&different_compression_loop);
1482 __ Ldrb(temp4, MemOperand(temp1.X(), c_char_size, PostIndex));
1483 __ Ldrh(temp3, MemOperand(temp2.X(), char_size, PostIndex));
1484 __ Subs(temp4, temp4, Operand(temp3));
1485 __ B(&different_compression_diff, ne);
1486 __ Subs(temp0, temp0, 2);
1487 __ B(&different_compression_loop, hi);
jessicahandojo05765752016-09-09 19:01:32 -07001488 __ B(&end);
1489
1490 // Calculate the difference.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001491 __ Bind(&different_compression_diff);
1492 __ Tst(temp0, Operand(1));
1493 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u,
1494 "Expecting 0=compressed, 1=uncompressed");
1495 __ Cneg(out, temp4, ne);
jessicahandojo05765752016-09-09 19:01:32 -07001496 }
Scott Wakeling1f36f412016-04-21 11:13:45 +01001497
1498 __ Bind(&end);
1499
1500 if (can_slow_path) {
1501 __ Bind(slow_path->GetExitLabel());
1502 }
Nicolas Geoffrayd75948a2015-03-27 09:53:16 +00001503}
1504
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001505// The cut off for unrolling the loop in String.equals() intrinsic for const strings.
1506// The normal loop plus the pre-header is 9 instructions without string compression and 12
1507// instructions with string compression. We can compare up to 8 bytes in 4 instructions
1508// (LDR+LDR+CMP+BNE) and up to 16 bytes in 5 instructions (LDP+LDP+CMP+CCMP+BNE). Allow up
1509// to 10 instructions for the unrolled loop.
1510constexpr size_t kShortConstStringEqualsCutoffInBytes = 32;
1511
1512static const char* GetConstString(HInstruction* candidate, uint32_t* utf16_length) {
1513 if (candidate->IsLoadString()) {
1514 HLoadString* load_string = candidate->AsLoadString();
1515 const DexFile& dex_file = load_string->GetDexFile();
1516 return dex_file.StringDataAndUtf16LengthByIdx(load_string->GetStringIndex(), utf16_length);
1517 }
1518 return nullptr;
1519}
1520
Agi Csakiea34b402015-08-13 17:51:19 -07001521void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001522 LocationSummary* locations =
1523 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Agi Csakiea34b402015-08-13 17:51:19 -07001524 locations->SetInAt(0, Location::RequiresRegister());
1525 locations->SetInAt(1, Location::RequiresRegister());
Agi Csakiea34b402015-08-13 17:51:19 -07001526
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001527 // For the generic implementation and for long const strings we need a temporary.
1528 // We do not need it for short const strings, up to 8 bytes, see code generation below.
1529 uint32_t const_string_length = 0u;
1530 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length);
1531 if (const_string == nullptr) {
1532 const_string = GetConstString(invoke->InputAt(1), &const_string_length);
1533 }
1534 bool is_compressed =
1535 mirror::kUseStringCompression &&
1536 const_string != nullptr &&
1537 mirror::String::DexFileStringAllASCII(const_string, const_string_length);
1538 if (const_string == nullptr || const_string_length > (is_compressed ? 8u : 4u)) {
1539 locations->AddTemp(Location::RequiresRegister());
1540 }
1541
1542 // TODO: If the String.equals() is used only for an immediately following HIf, we can
1543 // mark it as emitted-at-use-site and emit branches directly to the appropriate blocks.
1544 // Then we shall need an extra temporary register instead of the output register.
Agi Csakiea34b402015-08-13 17:51:19 -07001545 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1546}
1547
1548void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001549 MacroAssembler* masm = GetVIXLAssembler();
Agi Csakiea34b402015-08-13 17:51:19 -07001550 LocationSummary* locations = invoke->GetLocations();
1551
1552 Register str = WRegisterFrom(locations->InAt(0));
1553 Register arg = WRegisterFrom(locations->InAt(1));
1554 Register out = XRegisterFrom(locations->Out());
1555
1556 UseScratchRegisterScope scratch_scope(masm);
1557 Register temp = scratch_scope.AcquireW();
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001558 Register temp1 = scratch_scope.AcquireW();
Agi Csakiea34b402015-08-13 17:51:19 -07001559
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001560 vixl::aarch64::Label loop;
Scott Wakeling97c72b72016-06-24 16:19:36 +01001561 vixl::aarch64::Label end;
1562 vixl::aarch64::Label return_true;
1563 vixl::aarch64::Label return_false;
Agi Csakiea34b402015-08-13 17:51:19 -07001564
1565 // Get offsets of count, value, and class fields within a string object.
1566 const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1567 const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1568 const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1569
1570 // Note that the null check must have been done earlier.
1571 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1572
Vladimir Marko53b52002016-05-24 19:30:45 +01001573 StringEqualsOptimizations optimizations(invoke);
1574 if (!optimizations.GetArgumentNotNull()) {
1575 // Check if input is null, return false if it is.
1576 __ Cbz(arg, &return_false);
1577 }
Agi Csakiea34b402015-08-13 17:51:19 -07001578
1579 // Reference equality check, return true if same reference.
1580 __ Cmp(str, arg);
1581 __ B(&return_true, eq);
1582
Vladimir Marko53b52002016-05-24 19:30:45 +01001583 if (!optimizations.GetArgumentIsString()) {
1584 // Instanceof check for the argument by comparing class fields.
1585 // All string objects must have the same type since String cannot be subclassed.
1586 // Receiver must be a string object, so its class field is equal to all strings' class fields.
1587 // If the argument is a string object, its class field must be equal to receiver's class field.
1588 __ Ldr(temp, MemOperand(str.X(), class_offset));
1589 __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1590 __ Cmp(temp, temp1);
1591 __ B(&return_false, ne);
1592 }
Agi Csakiea34b402015-08-13 17:51:19 -07001593
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001594 // Check if one of the inputs is a const string. Do not special-case both strings
1595 // being const, such cases should be handled by constant folding if needed.
1596 uint32_t const_string_length = 0u;
1597 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length);
1598 if (const_string == nullptr) {
1599 const_string = GetConstString(invoke->InputAt(1), &const_string_length);
1600 if (const_string != nullptr) {
1601 std::swap(str, arg); // Make sure the const string is in `str`.
1602 }
1603 }
1604 bool is_compressed =
1605 mirror::kUseStringCompression &&
1606 const_string != nullptr &&
1607 mirror::String::DexFileStringAllASCII(const_string, const_string_length);
1608
1609 if (const_string != nullptr) {
1610 // Load `count` field of the argument string and check if it matches the const string.
1611 // Also compares the compression style, if differs return false.
1612 __ Ldr(temp, MemOperand(arg.X(), count_offset));
Vladimir Marko26ec3ca2017-03-14 13:37:14 +00001613 // Temporarily release temp1 as we may not be able to embed the flagged count in CMP immediate.
1614 scratch_scope.Release(temp1);
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001615 __ Cmp(temp, Operand(mirror::String::GetFlaggedCount(const_string_length, is_compressed)));
Vladimir Marko26ec3ca2017-03-14 13:37:14 +00001616 temp1 = scratch_scope.AcquireW();
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001617 __ B(&return_false, ne);
1618 } else {
1619 // Load `count` fields of this and argument strings.
1620 __ Ldr(temp, MemOperand(str.X(), count_offset));
1621 __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1622 // Check if `count` fields are equal, return false if they're not.
1623 // Also compares the compression style, if differs return false.
1624 __ Cmp(temp, temp1);
1625 __ B(&return_false, ne);
1626 }
Agi Csakiea34b402015-08-13 17:51:19 -07001627
Vladimir Markofdaf0f42016-10-13 19:29:53 +01001628 // Assertions that must hold in order to compare strings 8 bytes at a time.
Vladimir Marko984519c2017-08-23 10:45:29 +01001629 // Ok to do this because strings are zero-padded to kObjectAlignment.
Agi Csakiea34b402015-08-13 17:51:19 -07001630 DCHECK_ALIGNED(value_offset, 8);
1631 static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1632
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001633 if (const_string != nullptr &&
Vladimir Marko984519c2017-08-23 10:45:29 +01001634 const_string_length <= (is_compressed ? kShortConstStringEqualsCutoffInBytes
1635 : kShortConstStringEqualsCutoffInBytes / 2u)) {
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001636 // Load and compare the contents. Though we know the contents of the short const string
1637 // at compile time, materializing constants may be more code than loading from memory.
1638 int32_t offset = value_offset;
1639 size_t remaining_bytes =
1640 RoundUp(is_compressed ? const_string_length : const_string_length * 2u, 8u);
1641 temp = temp.X();
1642 temp1 = temp1.X();
Vladimir Marko984519c2017-08-23 10:45:29 +01001643 while (remaining_bytes > sizeof(uint64_t)) {
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001644 Register temp2 = XRegisterFrom(locations->GetTemp(0));
1645 __ Ldp(temp, temp1, MemOperand(str.X(), offset));
1646 __ Ldp(temp2, out, MemOperand(arg.X(), offset));
1647 __ Cmp(temp, temp2);
1648 __ Ccmp(temp1, out, NoFlag, eq);
1649 __ B(&return_false, ne);
1650 offset += 2u * sizeof(uint64_t);
1651 remaining_bytes -= 2u * sizeof(uint64_t);
1652 }
1653 if (remaining_bytes != 0u) {
1654 __ Ldr(temp, MemOperand(str.X(), offset));
1655 __ Ldr(temp1, MemOperand(arg.X(), offset));
1656 __ Cmp(temp, temp1);
1657 __ B(&return_false, ne);
1658 }
1659 } else {
1660 // Return true if both strings are empty. Even with string compression `count == 0` means empty.
1661 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u,
1662 "Expecting 0=compressed, 1=uncompressed");
1663 __ Cbz(temp, &return_true);
1664
1665 if (mirror::kUseStringCompression) {
1666 // For string compression, calculate the number of bytes to compare (not chars).
1667 // This could in theory exceed INT32_MAX, so treat temp as unsigned.
1668 __ And(temp1, temp, Operand(1)); // Extract compression flag.
1669 __ Lsr(temp, temp, 1u); // Extract length.
1670 __ Lsl(temp, temp, temp1); // Calculate number of bytes to compare.
1671 }
1672
1673 // Store offset of string value in preparation for comparison loop
1674 __ Mov(temp1, value_offset);
1675
1676 temp1 = temp1.X();
1677 Register temp2 = XRegisterFrom(locations->GetTemp(0));
1678 // Loop to compare strings 8 bytes at a time starting at the front of the string.
Vladimir Markoe39f14f2017-02-10 15:44:25 +00001679 __ Bind(&loop);
1680 __ Ldr(out, MemOperand(str.X(), temp1));
1681 __ Ldr(temp2, MemOperand(arg.X(), temp1));
1682 __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1683 __ Cmp(out, temp2);
1684 __ B(&return_false, ne);
1685 // With string compression, we have compared 8 bytes, otherwise 4 chars.
1686 __ Sub(temp, temp, Operand(mirror::kUseStringCompression ? 8 : 4), SetFlags);
1687 __ B(&loop, hi);
jessicahandojo05765752016-09-09 19:01:32 -07001688 }
1689
Agi Csakiea34b402015-08-13 17:51:19 -07001690 // Return true and exit the function.
1691 // If loop does not result in returning false, we return true.
1692 __ Bind(&return_true);
1693 __ Mov(out, 1);
1694 __ B(&end);
1695
1696 // Return false and exit the function.
1697 __ Bind(&return_false);
1698 __ Mov(out, 0);
1699 __ Bind(&end);
1700}
1701
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001702static void GenerateVisitStringIndexOf(HInvoke* invoke,
Scott Wakeling97c72b72016-06-24 16:19:36 +01001703 MacroAssembler* masm,
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001704 CodeGeneratorARM64* codegen,
1705 ArenaAllocator* allocator,
1706 bool start_at_zero) {
1707 LocationSummary* locations = invoke->GetLocations();
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001708
1709 // Note that the null check must have been done earlier.
1710 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1711
1712 // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001713 // or directly dispatch for a large constant, or omit slow-path for a small constant or a char.
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001714 SlowPathCodeARM64* slow_path = nullptr;
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001715 HInstruction* code_point = invoke->InputAt(1);
1716 if (code_point->IsIntConstant()) {
Vladimir Markoda051082016-05-17 16:10:20 +01001717 if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > 0xFFFFU) {
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001718 // Always needs the slow-path. We could directly dispatch to it, but this case should be
1719 // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1720 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1721 codegen->AddSlowPath(slow_path);
1722 __ B(slow_path->GetEntryLabel());
1723 __ Bind(slow_path->GetExitLabel());
1724 return;
1725 }
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001726 } else if (code_point->GetType() != DataType::Type::kUint16) {
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001727 Register char_reg = WRegisterFrom(locations->InAt(1));
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001728 __ Tst(char_reg, 0xFFFF0000);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001729 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1730 codegen->AddSlowPath(slow_path);
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001731 __ B(ne, slow_path->GetEntryLabel());
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001732 }
1733
1734 if (start_at_zero) {
1735 // Start-index = 0.
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001736 Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001737 __ Mov(tmp_reg, 0);
1738 }
1739
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001740 codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path);
Roland Levillain42ad2882016-02-29 18:26:54 +00001741 CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>();
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001742
1743 if (slow_path != nullptr) {
1744 __ Bind(slow_path->GetExitLabel());
1745 }
1746}
1747
1748void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001749 LocationSummary* locations = new (allocator_) LocationSummary(
1750 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001751 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1752 // best to align the inputs accordingly.
1753 InvokeRuntimeCallingConvention calling_convention;
1754 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1755 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001756 locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kInt32));
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001757
Vladimir Markofb6c90a2016-05-06 15:52:12 +01001758 // Need to send start_index=0.
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001759 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1760}
1761
1762void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001763 GenerateVisitStringIndexOf(
1764 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001765}
1766
1767void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001768 LocationSummary* locations = new (allocator_) LocationSummary(
1769 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001770 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1771 // best to align the inputs accordingly.
1772 InvokeRuntimeCallingConvention calling_convention;
1773 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1774 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1775 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001776 locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kInt32));
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001777}
1778
1779void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
Roland Levillainbf84a3d2015-12-04 14:33:02 +00001780 GenerateVisitStringIndexOf(
1781 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
Andreas Gampeba6fdbc2015-05-07 22:31:55 -07001782}
1783
Jeff Hao848f70a2014-01-15 13:49:50 -08001784void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001785 LocationSummary* locations = new (allocator_) LocationSummary(
1786 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified);
Jeff Hao848f70a2014-01-15 13:49:50 -08001787 InvokeRuntimeCallingConvention calling_convention;
1788 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1789 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1790 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1791 locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001792 locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kReference));
Jeff Hao848f70a2014-01-15 13:49:50 -08001793}
1794
1795void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001796 MacroAssembler* masm = GetVIXLAssembler();
Jeff Hao848f70a2014-01-15 13:49:50 -08001797 LocationSummary* locations = invoke->GetLocations();
1798
1799 Register byte_array = WRegisterFrom(locations->InAt(0));
1800 __ Cmp(byte_array, 0);
1801 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1802 codegen_->AddSlowPath(slow_path);
1803 __ B(eq, slow_path->GetEntryLabel());
1804
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001805 codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc(), slow_path);
Roland Levillainf969a202016-03-09 16:14:00 +00001806 CheckEntrypointTypes<kQuickAllocStringFromBytes, void*, void*, int32_t, int32_t, int32_t>();
Jeff Hao848f70a2014-01-15 13:49:50 -08001807 __ Bind(slow_path->GetExitLabel());
1808}
1809
1810void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001811 LocationSummary* locations =
1812 new (allocator_) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified);
Jeff Hao848f70a2014-01-15 13:49:50 -08001813 InvokeRuntimeCallingConvention calling_convention;
1814 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1815 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1816 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001817 locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kReference));
Jeff Hao848f70a2014-01-15 13:49:50 -08001818}
1819
1820void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
Roland Levillaincc3839c2016-02-29 16:23:48 +00001821 // No need to emit code checking whether `locations->InAt(2)` is a null
1822 // pointer, as callers of the native method
1823 //
1824 // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data)
1825 //
1826 // all include a null check on `data` before calling that method.
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001827 codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc());
Roland Levillainf969a202016-03-09 16:14:00 +00001828 CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>();
Jeff Hao848f70a2014-01-15 13:49:50 -08001829}
1830
1831void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001832 LocationSummary* locations = new (allocator_) LocationSummary(
1833 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified);
Jeff Hao848f70a2014-01-15 13:49:50 -08001834 InvokeRuntimeCallingConvention calling_convention;
1835 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001836 locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kReference));
Jeff Hao848f70a2014-01-15 13:49:50 -08001837}
1838
1839void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01001840 MacroAssembler* masm = GetVIXLAssembler();
Jeff Hao848f70a2014-01-15 13:49:50 -08001841 LocationSummary* locations = invoke->GetLocations();
1842
1843 Register string_to_copy = WRegisterFrom(locations->InAt(0));
1844 __ Cmp(string_to_copy, 0);
1845 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1846 codegen_->AddSlowPath(slow_path);
1847 __ B(eq, slow_path->GetEntryLabel());
1848
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001849 codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc(), slow_path);
Roland Levillainf969a202016-03-09 16:14:00 +00001850 CheckEntrypointTypes<kQuickAllocStringFromString, void*, void*>();
Jeff Hao848f70a2014-01-15 13:49:50 -08001851 __ Bind(slow_path->GetExitLabel());
1852}
1853
Vladimir Markoca6fff82017-10-03 14:49:14 +01001854static void CreateFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) {
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001855 DCHECK_EQ(invoke->GetNumberOfArguments(), 1U);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001856 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1857 DCHECK(DataType::IsFloatingPointType(invoke->GetType()));
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001858
Vladimir Markoca6fff82017-10-03 14:49:14 +01001859 LocationSummary* const locations =
1860 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001861 InvokeRuntimeCallingConvention calling_convention;
1862
1863 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1864 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1865}
1866
Vladimir Markoca6fff82017-10-03 14:49:14 +01001867static void CreateFPFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) {
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001868 DCHECK_EQ(invoke->GetNumberOfArguments(), 2U);
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01001869 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1870 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(1)->GetType()));
1871 DCHECK(DataType::IsFloatingPointType(invoke->GetType()));
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001872
Vladimir Markoca6fff82017-10-03 14:49:14 +01001873 LocationSummary* const locations =
1874 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001875 InvokeRuntimeCallingConvention calling_convention;
1876
1877 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1878 locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
1879 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1880}
1881
1882static void GenFPToFPCall(HInvoke* invoke,
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001883 CodeGeneratorARM64* codegen,
1884 QuickEntrypointEnum entry) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001885 codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc());
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001886}
1887
1888void IntrinsicLocationsBuilderARM64::VisitMathCos(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001889 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001890}
1891
1892void IntrinsicCodeGeneratorARM64::VisitMathCos(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001893 GenFPToFPCall(invoke, codegen_, kQuickCos);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001894}
1895
1896void IntrinsicLocationsBuilderARM64::VisitMathSin(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001897 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001898}
1899
1900void IntrinsicCodeGeneratorARM64::VisitMathSin(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001901 GenFPToFPCall(invoke, codegen_, kQuickSin);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001902}
1903
1904void IntrinsicLocationsBuilderARM64::VisitMathAcos(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001905 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001906}
1907
1908void IntrinsicCodeGeneratorARM64::VisitMathAcos(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001909 GenFPToFPCall(invoke, codegen_, kQuickAcos);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001910}
1911
1912void IntrinsicLocationsBuilderARM64::VisitMathAsin(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001913 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001914}
1915
1916void IntrinsicCodeGeneratorARM64::VisitMathAsin(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001917 GenFPToFPCall(invoke, codegen_, kQuickAsin);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001918}
1919
1920void IntrinsicLocationsBuilderARM64::VisitMathAtan(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001921 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001922}
1923
1924void IntrinsicCodeGeneratorARM64::VisitMathAtan(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001925 GenFPToFPCall(invoke, codegen_, kQuickAtan);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001926}
1927
1928void IntrinsicLocationsBuilderARM64::VisitMathCbrt(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001929 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001930}
1931
1932void IntrinsicCodeGeneratorARM64::VisitMathCbrt(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001933 GenFPToFPCall(invoke, codegen_, kQuickCbrt);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001934}
1935
1936void IntrinsicLocationsBuilderARM64::VisitMathCosh(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001937 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001938}
1939
1940void IntrinsicCodeGeneratorARM64::VisitMathCosh(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001941 GenFPToFPCall(invoke, codegen_, kQuickCosh);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001942}
1943
1944void IntrinsicLocationsBuilderARM64::VisitMathExp(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001945 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001946}
1947
1948void IntrinsicCodeGeneratorARM64::VisitMathExp(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001949 GenFPToFPCall(invoke, codegen_, kQuickExp);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001950}
1951
1952void IntrinsicLocationsBuilderARM64::VisitMathExpm1(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001953 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001954}
1955
1956void IntrinsicCodeGeneratorARM64::VisitMathExpm1(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001957 GenFPToFPCall(invoke, codegen_, kQuickExpm1);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001958}
1959
1960void IntrinsicLocationsBuilderARM64::VisitMathLog(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001961 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001962}
1963
1964void IntrinsicCodeGeneratorARM64::VisitMathLog(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001965 GenFPToFPCall(invoke, codegen_, kQuickLog);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001966}
1967
1968void IntrinsicLocationsBuilderARM64::VisitMathLog10(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001969 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001970}
1971
1972void IntrinsicCodeGeneratorARM64::VisitMathLog10(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001973 GenFPToFPCall(invoke, codegen_, kQuickLog10);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001974}
1975
1976void IntrinsicLocationsBuilderARM64::VisitMathSinh(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001977 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001978}
1979
1980void IntrinsicCodeGeneratorARM64::VisitMathSinh(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001981 GenFPToFPCall(invoke, codegen_, kQuickSinh);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001982}
1983
1984void IntrinsicLocationsBuilderARM64::VisitMathTan(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001985 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001986}
1987
1988void IntrinsicCodeGeneratorARM64::VisitMathTan(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001989 GenFPToFPCall(invoke, codegen_, kQuickTan);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001990}
1991
1992void IntrinsicLocationsBuilderARM64::VisitMathTanh(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01001993 CreateFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001994}
1995
1996void IntrinsicCodeGeneratorARM64::VisitMathTanh(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00001997 GenFPToFPCall(invoke, codegen_, kQuickTanh);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00001998}
1999
2000void IntrinsicLocationsBuilderARM64::VisitMathAtan2(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01002001 CreateFPFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00002002}
2003
2004void IntrinsicCodeGeneratorARM64::VisitMathAtan2(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00002005 GenFPToFPCall(invoke, codegen_, kQuickAtan2);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00002006}
2007
2008void IntrinsicLocationsBuilderARM64::VisitMathHypot(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01002009 CreateFPFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00002010}
2011
2012void IntrinsicCodeGeneratorARM64::VisitMathHypot(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00002013 GenFPToFPCall(invoke, codegen_, kQuickHypot);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00002014}
2015
2016void IntrinsicLocationsBuilderARM64::VisitMathNextAfter(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01002017 CreateFPFPToFPCallLocations(allocator_, invoke);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00002018}
2019
2020void IntrinsicCodeGeneratorARM64::VisitMathNextAfter(HInvoke* invoke) {
Serban Constantinescu22f81d32016-02-18 16:06:31 +00002021 GenFPToFPCall(invoke, codegen_, kQuickNextAfter);
Anton Kirilov02fc24e2016-01-20 16:48:19 +00002022}
2023
Tim Zhang25abd6c2016-01-19 23:39:24 +08002024void IntrinsicLocationsBuilderARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01002025 LocationSummary* locations =
2026 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Tim Zhang25abd6c2016-01-19 23:39:24 +08002027 locations->SetInAt(0, Location::RequiresRegister());
2028 locations->SetInAt(1, Location::RequiresRegister());
2029 locations->SetInAt(2, Location::RequiresRegister());
2030 locations->SetInAt(3, Location::RequiresRegister());
2031 locations->SetInAt(4, Location::RequiresRegister());
2032
2033 locations->AddTemp(Location::RequiresRegister());
2034 locations->AddTemp(Location::RequiresRegister());
Scott Wakelingdf109d92016-04-22 11:35:56 +01002035 locations->AddTemp(Location::RequiresRegister());
Tim Zhang25abd6c2016-01-19 23:39:24 +08002036}
2037
2038void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01002039 MacroAssembler* masm = GetVIXLAssembler();
Tim Zhang25abd6c2016-01-19 23:39:24 +08002040 LocationSummary* locations = invoke->GetLocations();
2041
2042 // Check assumption that sizeof(Char) is 2 (used in scaling below).
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002043 const size_t char_size = DataType::Size(DataType::Type::kUint16);
Tim Zhang25abd6c2016-01-19 23:39:24 +08002044 DCHECK_EQ(char_size, 2u);
2045
2046 // Location of data in char array buffer.
2047 const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value();
2048
2049 // Location of char array data in string.
2050 const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value();
2051
2052 // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin);
2053 // Since getChars() calls getCharsNoCheck() - we use registers rather than constants.
2054 Register srcObj = XRegisterFrom(locations->InAt(0));
2055 Register srcBegin = XRegisterFrom(locations->InAt(1));
2056 Register srcEnd = XRegisterFrom(locations->InAt(2));
2057 Register dstObj = XRegisterFrom(locations->InAt(3));
2058 Register dstBegin = XRegisterFrom(locations->InAt(4));
2059
2060 Register src_ptr = XRegisterFrom(locations->GetTemp(0));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002061 Register num_chr = XRegisterFrom(locations->GetTemp(1));
2062 Register tmp1 = XRegisterFrom(locations->GetTemp(2));
Tim Zhang25abd6c2016-01-19 23:39:24 +08002063
2064 UseScratchRegisterScope temps(masm);
2065 Register dst_ptr = temps.AcquireX();
Scott Wakelingdf109d92016-04-22 11:35:56 +01002066 Register tmp2 = temps.AcquireX();
Tim Zhang25abd6c2016-01-19 23:39:24 +08002067
jessicahandojo05765752016-09-09 19:01:32 -07002068 vixl::aarch64::Label done;
2069 vixl::aarch64::Label compressed_string_loop;
2070 __ Sub(num_chr, srcEnd, srcBegin);
2071 // Early out for valid zero-length retrievals.
2072 __ Cbz(num_chr, &done);
Tim Zhang25abd6c2016-01-19 23:39:24 +08002073
Scott Wakelingdf109d92016-04-22 11:35:56 +01002074 // dst address start to copy to.
Tim Zhang25abd6c2016-01-19 23:39:24 +08002075 __ Add(dst_ptr, dstObj, Operand(data_offset));
2076 __ Add(dst_ptr, dst_ptr, Operand(dstBegin, LSL, 1));
2077
jessicahandojo05765752016-09-09 19:01:32 -07002078 // src address to copy from.
2079 __ Add(src_ptr, srcObj, Operand(value_offset));
2080 vixl::aarch64::Label compressed_string_preloop;
2081 if (mirror::kUseStringCompression) {
2082 // Location of count in string.
2083 const uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
2084 // String's length.
Vladimir Markofdaf0f42016-10-13 19:29:53 +01002085 __ Ldr(tmp2, MemOperand(srcObj, count_offset));
2086 __ Tbz(tmp2, 0, &compressed_string_preloop);
jessicahandojo05765752016-09-09 19:01:32 -07002087 }
2088 __ Add(src_ptr, src_ptr, Operand(srcBegin, LSL, 1));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002089
Tim Zhang25abd6c2016-01-19 23:39:24 +08002090 // Do the copy.
Scott Wakeling97c72b72016-06-24 16:19:36 +01002091 vixl::aarch64::Label loop;
Scott Wakeling97c72b72016-06-24 16:19:36 +01002092 vixl::aarch64::Label remainder;
Scott Wakelingdf109d92016-04-22 11:35:56 +01002093
Scott Wakelingdf109d92016-04-22 11:35:56 +01002094 // Save repairing the value of num_chr on the < 8 character path.
2095 __ Subs(tmp1, num_chr, 8);
2096 __ B(lt, &remainder);
2097
2098 // Keep the result of the earlier subs, we are going to fetch at least 8 characters.
2099 __ Mov(num_chr, tmp1);
2100
2101 // Main loop used for longer fetches loads and stores 8x16-bit characters at a time.
2102 // (Unaligned addresses are acceptable here and not worth inlining extra code to rectify.)
Tim Zhang25abd6c2016-01-19 23:39:24 +08002103 __ Bind(&loop);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002104 __ Ldp(tmp1, tmp2, MemOperand(src_ptr, char_size * 8, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002105 __ Subs(num_chr, num_chr, 8);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002106 __ Stp(tmp1, tmp2, MemOperand(dst_ptr, char_size * 8, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002107 __ B(ge, &loop);
2108
2109 __ Adds(num_chr, num_chr, 8);
2110 __ B(eq, &done);
2111
2112 // Main loop for < 8 character case and remainder handling. Loads and stores one
2113 // 16-bit Java character at a time.
2114 __ Bind(&remainder);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002115 __ Ldrh(tmp1, MemOperand(src_ptr, char_size, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002116 __ Subs(num_chr, num_chr, 1);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002117 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex));
Scott Wakelingdf109d92016-04-22 11:35:56 +01002118 __ B(gt, &remainder);
jessicahandojo05765752016-09-09 19:01:32 -07002119 __ B(&done);
2120
2121 if (mirror::kUseStringCompression) {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002122 const size_t c_char_size = DataType::Size(DataType::Type::kInt8);
jessicahandojo05765752016-09-09 19:01:32 -07002123 DCHECK_EQ(c_char_size, 1u);
2124 __ Bind(&compressed_string_preloop);
2125 __ Add(src_ptr, src_ptr, Operand(srcBegin));
2126 // Copy loop for compressed src, copying 1 character (8-bit) to (16-bit) at a time.
2127 __ Bind(&compressed_string_loop);
2128 __ Ldrb(tmp1, MemOperand(src_ptr, c_char_size, PostIndex));
2129 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex));
2130 __ Subs(num_chr, num_chr, Operand(1));
2131 __ B(gt, &compressed_string_loop);
2132 }
Scott Wakelingdf109d92016-04-22 11:35:56 +01002133
Tim Zhang25abd6c2016-01-19 23:39:24 +08002134 __ Bind(&done);
2135}
2136
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002137// Mirrors ARRAYCOPY_SHORT_CHAR_ARRAY_THRESHOLD in libcore, so we can choose to use the native
2138// implementation there for longer copy lengths.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002139static constexpr int32_t kSystemArrayCopyCharThreshold = 32;
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002140
2141static void SetSystemArrayCopyLocationRequires(LocationSummary* locations,
2142 uint32_t at,
2143 HInstruction* input) {
2144 HIntConstant* const_input = input->AsIntConstant();
Scott Wakeling97c72b72016-06-24 16:19:36 +01002145 if (const_input != nullptr && !vixl::aarch64::Assembler::IsImmAddSub(const_input->GetValue())) {
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002146 locations->SetInAt(at, Location::RequiresRegister());
2147 } else {
2148 locations->SetInAt(at, Location::RegisterOrConstant(input));
2149 }
2150}
2151
2152void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopyChar(HInvoke* invoke) {
2153 // Check to see if we have known failures that will cause us to have to bail out
2154 // to the runtime, and just generate the runtime call directly.
2155 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
2156 HIntConstant* dst_pos = invoke->InputAt(3)->AsIntConstant();
2157
2158 // The positions must be non-negative.
2159 if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
2160 (dst_pos != nullptr && dst_pos->GetValue() < 0)) {
2161 // We will have to fail anyways.
2162 return;
2163 }
2164
2165 // The length must be >= 0 and not so long that we would (currently) prefer libcore's
2166 // native implementation.
2167 HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
2168 if (length != nullptr) {
2169 int32_t len = length->GetValue();
donghui.baic2ec9ad2016-03-10 14:02:55 +08002170 if (len < 0 || len > kSystemArrayCopyCharThreshold) {
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002171 // Just call as normal.
2172 return;
2173 }
2174 }
2175
Vladimir Markoca6fff82017-10-03 14:49:14 +01002176 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator();
2177 LocationSummary* locations =
2178 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002179 // arraycopy(char[] src, int src_pos, char[] dst, int dst_pos, int length).
2180 locations->SetInAt(0, Location::RequiresRegister());
2181 SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1));
2182 locations->SetInAt(2, Location::RequiresRegister());
2183 SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3));
2184 SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4));
2185
2186 locations->AddTemp(Location::RequiresRegister());
2187 locations->AddTemp(Location::RequiresRegister());
2188 locations->AddTemp(Location::RequiresRegister());
2189}
2190
Scott Wakeling97c72b72016-06-24 16:19:36 +01002191static void CheckSystemArrayCopyPosition(MacroAssembler* masm,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002192 const Location& pos,
2193 const Register& input,
2194 const Location& length,
2195 SlowPathCodeARM64* slow_path,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002196 const Register& temp,
2197 bool length_is_input_length = false) {
2198 const int32_t length_offset = mirror::Array::LengthOffset().Int32Value();
2199 if (pos.IsConstant()) {
2200 int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue();
2201 if (pos_const == 0) {
2202 if (!length_is_input_length) {
2203 // Check that length(input) >= length.
2204 __ Ldr(temp, MemOperand(input, length_offset));
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002205 __ Cmp(temp, OperandFrom(length, DataType::Type::kInt32));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002206 __ B(slow_path->GetEntryLabel(), lt);
2207 }
2208 } else {
2209 // Check that length(input) >= pos.
Nicolas Geoffrayfea1abd2016-07-06 12:09:12 +01002210 __ Ldr(temp, MemOperand(input, length_offset));
2211 __ Subs(temp, temp, pos_const);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002212 __ B(slow_path->GetEntryLabel(), lt);
2213
2214 // Check that (length(input) - pos) >= length.
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002215 __ Cmp(temp, OperandFrom(length, DataType::Type::kInt32));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002216 __ B(slow_path->GetEntryLabel(), lt);
2217 }
2218 } else if (length_is_input_length) {
2219 // The only way the copy can succeed is if pos is zero.
2220 __ Cbnz(WRegisterFrom(pos), slow_path->GetEntryLabel());
2221 } else {
2222 // Check that pos >= 0.
2223 Register pos_reg = WRegisterFrom(pos);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002224 __ Tbnz(pos_reg, pos_reg.GetSizeInBits() - 1, slow_path->GetEntryLabel());
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002225
2226 // Check that pos <= length(input) && (length(input) - pos) >= length.
2227 __ Ldr(temp, MemOperand(input, length_offset));
2228 __ Subs(temp, temp, pos_reg);
2229 // Ccmp if length(input) >= pos, else definitely bail to slow path (N!=V == lt).
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002230 __ Ccmp(temp, OperandFrom(length, DataType::Type::kInt32), NFlag, ge);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002231 __ B(slow_path->GetEntryLabel(), lt);
2232 }
2233}
2234
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002235// Compute base source address, base destination address, and end
2236// source address for System.arraycopy* intrinsics in `src_base`,
2237// `dst_base` and `src_end` respectively.
Scott Wakeling97c72b72016-06-24 16:19:36 +01002238static void GenSystemArrayCopyAddresses(MacroAssembler* masm,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002239 DataType::Type type,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002240 const Register& src,
2241 const Location& src_pos,
2242 const Register& dst,
2243 const Location& dst_pos,
2244 const Location& copy_length,
2245 const Register& src_base,
2246 const Register& dst_base,
2247 const Register& src_end) {
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002248 // This routine is used by the SystemArrayCopy and the SystemArrayCopyChar intrinsics.
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002249 DCHECK(type == DataType::Type::kReference || type == DataType::Type::kUint16)
Roland Levillainebea3d22016-04-12 15:42:57 +01002250 << "Unexpected element type: " << type;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002251 const int32_t element_size = DataType::Size(type);
2252 const int32_t element_size_shift = DataType::SizeShift(type);
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002253 const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002254
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002255 if (src_pos.IsConstant()) {
2256 int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
Roland Levillainebea3d22016-04-12 15:42:57 +01002257 __ Add(src_base, src, element_size * constant + data_offset);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002258 } else {
Roland Levillainebea3d22016-04-12 15:42:57 +01002259 __ Add(src_base, src, data_offset);
2260 __ Add(src_base, src_base, Operand(XRegisterFrom(src_pos), LSL, element_size_shift));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002261 }
2262
2263 if (dst_pos.IsConstant()) {
2264 int32_t constant = dst_pos.GetConstant()->AsIntConstant()->GetValue();
Roland Levillainebea3d22016-04-12 15:42:57 +01002265 __ Add(dst_base, dst, element_size * constant + data_offset);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002266 } else {
Roland Levillainebea3d22016-04-12 15:42:57 +01002267 __ Add(dst_base, dst, data_offset);
2268 __ Add(dst_base, dst_base, Operand(XRegisterFrom(dst_pos), LSL, element_size_shift));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002269 }
2270
2271 if (copy_length.IsConstant()) {
2272 int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue();
Roland Levillainebea3d22016-04-12 15:42:57 +01002273 __ Add(src_end, src_base, element_size * constant);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002274 } else {
Roland Levillainebea3d22016-04-12 15:42:57 +01002275 __ Add(src_end, src_base, Operand(XRegisterFrom(copy_length), LSL, element_size_shift));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002276 }
2277}
2278
2279void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopyChar(HInvoke* invoke) {
Scott Wakeling97c72b72016-06-24 16:19:36 +01002280 MacroAssembler* masm = GetVIXLAssembler();
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002281 LocationSummary* locations = invoke->GetLocations();
2282 Register src = XRegisterFrom(locations->InAt(0));
2283 Location src_pos = locations->InAt(1);
2284 Register dst = XRegisterFrom(locations->InAt(2));
2285 Location dst_pos = locations->InAt(3);
2286 Location length = locations->InAt(4);
2287
2288 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
2289 codegen_->AddSlowPath(slow_path);
2290
2291 // If source and destination are the same, take the slow path. Overlapping copy regions must be
2292 // copied in reverse and we can't know in all cases if it's needed.
2293 __ Cmp(src, dst);
2294 __ B(slow_path->GetEntryLabel(), eq);
2295
2296 // Bail out if the source is null.
2297 __ Cbz(src, slow_path->GetEntryLabel());
2298
2299 // Bail out if the destination is null.
2300 __ Cbz(dst, slow_path->GetEntryLabel());
2301
2302 if (!length.IsConstant()) {
Vladimir Markoc5646202016-11-28 16:03:15 +00002303 // Merge the following two comparisons into one:
2304 // If the length is negative, bail out (delegate to libcore's native implementation).
2305 // If the length > 32 then (currently) prefer libcore's native implementation.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002306 __ Cmp(WRegisterFrom(length), kSystemArrayCopyCharThreshold);
Vladimir Markoc5646202016-11-28 16:03:15 +00002307 __ B(slow_path->GetEntryLabel(), hi);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002308 } else {
2309 // We have already checked in the LocationsBuilder for the constant case.
2310 DCHECK_GE(length.GetConstant()->AsIntConstant()->GetValue(), 0);
2311 DCHECK_LE(length.GetConstant()->AsIntConstant()->GetValue(), 32);
2312 }
2313
2314 Register src_curr_addr = WRegisterFrom(locations->GetTemp(0));
2315 Register dst_curr_addr = WRegisterFrom(locations->GetTemp(1));
2316 Register src_stop_addr = WRegisterFrom(locations->GetTemp(2));
2317
2318 CheckSystemArrayCopyPosition(masm,
2319 src_pos,
2320 src,
2321 length,
2322 slow_path,
2323 src_curr_addr,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002324 false);
2325
2326 CheckSystemArrayCopyPosition(masm,
2327 dst_pos,
2328 dst,
2329 length,
2330 slow_path,
2331 src_curr_addr,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002332 false);
2333
2334 src_curr_addr = src_curr_addr.X();
2335 dst_curr_addr = dst_curr_addr.X();
2336 src_stop_addr = src_stop_addr.X();
2337
2338 GenSystemArrayCopyAddresses(masm,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002339 DataType::Type::kUint16,
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002340 src,
2341 src_pos,
2342 dst,
2343 dst_pos,
2344 length,
2345 src_curr_addr,
2346 dst_curr_addr,
2347 src_stop_addr);
2348
2349 // Iterate over the arrays and do a raw copy of the chars.
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002350 const int32_t char_size = DataType::Size(DataType::Type::kUint16);
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002351 UseScratchRegisterScope temps(masm);
2352 Register tmp = temps.AcquireW();
Scott Wakeling97c72b72016-06-24 16:19:36 +01002353 vixl::aarch64::Label loop, done;
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002354 __ Bind(&loop);
2355 __ Cmp(src_curr_addr, src_stop_addr);
2356 __ B(&done, eq);
Scott Wakeling97c72b72016-06-24 16:19:36 +01002357 __ Ldrh(tmp, MemOperand(src_curr_addr, char_size, PostIndex));
2358 __ Strh(tmp, MemOperand(dst_curr_addr, char_size, PostIndex));
Scott Wakelingd3d0da52016-02-29 15:17:20 +00002359 __ B(&loop);
2360 __ Bind(&done);
2361
2362 __ Bind(slow_path->GetExitLabel());
2363}
2364
donghui.baic2ec9ad2016-03-10 14:02:55 +08002365// We can choose to use the native implementation there for longer copy lengths.
2366static constexpr int32_t kSystemArrayCopyThreshold = 128;
2367
2368// CodeGenerator::CreateSystemArrayCopyLocationSummary use three temporary registers.
2369// We want to use two temporary registers in order to reduce the register pressure in arm64.
2370// So we don't use the CodeGenerator::CreateSystemArrayCopyLocationSummary.
2371void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopy(HInvoke* invoke) {
Roland Levillain0b671c02016-08-19 12:02:34 +01002372 // The only read barrier implementation supporting the
2373 // SystemArrayCopy intrinsic is the Baker-style read barriers.
2374 if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) {
Roland Levillain3d312422016-06-23 13:53:42 +01002375 return;
2376 }
2377
donghui.baic2ec9ad2016-03-10 14:02:55 +08002378 // Check to see if we have known failures that will cause us to have to bail out
2379 // to the runtime, and just generate the runtime call directly.
2380 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
2381 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
2382
2383 // The positions must be non-negative.
2384 if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
2385 (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
2386 // We will have to fail anyways.
2387 return;
2388 }
2389
2390 // The length must be >= 0.
2391 HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
2392 if (length != nullptr) {
2393 int32_t len = length->GetValue();
2394 if (len < 0 || len >= kSystemArrayCopyThreshold) {
2395 // Just call as normal.
2396 return;
2397 }
2398 }
2399
2400 SystemArrayCopyOptimizations optimizations(invoke);
2401
2402 if (optimizations.GetDestinationIsSource()) {
2403 if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
2404 // We only support backward copying if source and destination are the same.
2405 return;
2406 }
2407 }
2408
2409 if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
2410 // We currently don't intrinsify primitive copying.
2411 return;
2412 }
2413
Vladimir Markoca6fff82017-10-03 14:49:14 +01002414 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator();
2415 LocationSummary* locations =
2416 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002417 // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
2418 locations->SetInAt(0, Location::RequiresRegister());
2419 SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1));
2420 locations->SetInAt(2, Location::RequiresRegister());
2421 SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3));
2422 SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4));
2423
2424 locations->AddTemp(Location::RequiresRegister());
2425 locations->AddTemp(Location::RequiresRegister());
Roland Levillain0b671c02016-08-19 12:02:34 +01002426 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2427 // Temporary register IP0, obtained from the VIXL scratch register
2428 // pool, cannot be used in ReadBarrierSystemArrayCopySlowPathARM64
2429 // (because that register is clobbered by ReadBarrierMarkRegX
Roland Levillain54f869e2017-03-06 13:54:11 +00002430 // entry points). It cannot be used in calls to
2431 // CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier
2432 // either. For these reasons, get a third extra temporary register
2433 // from the register allocator.
Roland Levillain0b671c02016-08-19 12:02:34 +01002434 locations->AddTemp(Location::RequiresRegister());
Roland Levillain54f869e2017-03-06 13:54:11 +00002435 } else {
2436 // Cases other than Baker read barriers: the third temporary will
2437 // be acquired from the VIXL scratch register pool.
Roland Levillain0b671c02016-08-19 12:02:34 +01002438 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002439}
2440
2441void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) {
Roland Levillain0b671c02016-08-19 12:02:34 +01002442 // The only read barrier implementation supporting the
2443 // SystemArrayCopy intrinsic is the Baker-style read barriers.
2444 DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier);
Roland Levillain3d312422016-06-23 13:53:42 +01002445
Scott Wakeling97c72b72016-06-24 16:19:36 +01002446 MacroAssembler* masm = GetVIXLAssembler();
donghui.baic2ec9ad2016-03-10 14:02:55 +08002447 LocationSummary* locations = invoke->GetLocations();
2448
2449 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2450 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2451 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2452 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
Roland Levillain0b671c02016-08-19 12:02:34 +01002453 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
donghui.baic2ec9ad2016-03-10 14:02:55 +08002454
2455 Register src = XRegisterFrom(locations->InAt(0));
2456 Location src_pos = locations->InAt(1);
2457 Register dest = XRegisterFrom(locations->InAt(2));
2458 Location dest_pos = locations->InAt(3);
2459 Location length = locations->InAt(4);
2460 Register temp1 = WRegisterFrom(locations->GetTemp(0));
Roland Levillain0b671c02016-08-19 12:02:34 +01002461 Location temp1_loc = LocationFrom(temp1);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002462 Register temp2 = WRegisterFrom(locations->GetTemp(1));
Roland Levillain0b671c02016-08-19 12:02:34 +01002463 Location temp2_loc = LocationFrom(temp2);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002464
Roland Levillain0b671c02016-08-19 12:02:34 +01002465 SlowPathCodeARM64* intrinsic_slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
2466 codegen_->AddSlowPath(intrinsic_slow_path);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002467
Scott Wakeling97c72b72016-06-24 16:19:36 +01002468 vixl::aarch64::Label conditions_on_positions_validated;
donghui.baic2ec9ad2016-03-10 14:02:55 +08002469 SystemArrayCopyOptimizations optimizations(invoke);
2470
donghui.baic2ec9ad2016-03-10 14:02:55 +08002471 // If source and destination are the same, we go to slow path if we need to do
2472 // forward copying.
2473 if (src_pos.IsConstant()) {
2474 int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
2475 if (dest_pos.IsConstant()) {
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002476 int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
2477 if (optimizations.GetDestinationIsSource()) {
2478 // Checked when building locations.
2479 DCHECK_GE(src_pos_constant, dest_pos_constant);
2480 } else if (src_pos_constant < dest_pos_constant) {
2481 __ Cmp(src, dest);
Roland Levillain0b671c02016-08-19 12:02:34 +01002482 __ B(intrinsic_slow_path->GetEntryLabel(), eq);
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002483 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002484 // Checked when building locations.
2485 DCHECK(!optimizations.GetDestinationIsSource()
2486 || (src_pos_constant >= dest_pos.GetConstant()->AsIntConstant()->GetValue()));
2487 } else {
2488 if (!optimizations.GetDestinationIsSource()) {
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002489 __ Cmp(src, dest);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002490 __ B(&conditions_on_positions_validated, ne);
2491 }
2492 __ Cmp(WRegisterFrom(dest_pos), src_pos_constant);
Roland Levillain0b671c02016-08-19 12:02:34 +01002493 __ B(intrinsic_slow_path->GetEntryLabel(), gt);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002494 }
2495 } else {
2496 if (!optimizations.GetDestinationIsSource()) {
Nicolas Geoffray9f65db82016-07-07 12:07:42 +01002497 __ Cmp(src, dest);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002498 __ B(&conditions_on_positions_validated, ne);
2499 }
2500 __ Cmp(RegisterFrom(src_pos, invoke->InputAt(1)->GetType()),
2501 OperandFrom(dest_pos, invoke->InputAt(3)->GetType()));
Roland Levillain0b671c02016-08-19 12:02:34 +01002502 __ B(intrinsic_slow_path->GetEntryLabel(), lt);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002503 }
2504
2505 __ Bind(&conditions_on_positions_validated);
2506
2507 if (!optimizations.GetSourceIsNotNull()) {
2508 // Bail out if the source is null.
Roland Levillain0b671c02016-08-19 12:02:34 +01002509 __ Cbz(src, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002510 }
2511
2512 if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) {
2513 // Bail out if the destination is null.
Roland Levillain0b671c02016-08-19 12:02:34 +01002514 __ Cbz(dest, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002515 }
2516
2517 // We have already checked in the LocationsBuilder for the constant case.
2518 if (!length.IsConstant() &&
2519 !optimizations.GetCountIsSourceLength() &&
2520 !optimizations.GetCountIsDestinationLength()) {
Vladimir Markoc5646202016-11-28 16:03:15 +00002521 // Merge the following two comparisons into one:
2522 // If the length is negative, bail out (delegate to libcore's native implementation).
2523 // If the length >= 128 then (currently) prefer native implementation.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002524 __ Cmp(WRegisterFrom(length), kSystemArrayCopyThreshold);
Vladimir Markoc5646202016-11-28 16:03:15 +00002525 __ B(intrinsic_slow_path->GetEntryLabel(), hs);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002526 }
2527 // Validity checks: source.
2528 CheckSystemArrayCopyPosition(masm,
2529 src_pos,
2530 src,
2531 length,
Roland Levillain0b671c02016-08-19 12:02:34 +01002532 intrinsic_slow_path,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002533 temp1,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002534 optimizations.GetCountIsSourceLength());
2535
2536 // Validity checks: dest.
2537 CheckSystemArrayCopyPosition(masm,
2538 dest_pos,
2539 dest,
2540 length,
Roland Levillain0b671c02016-08-19 12:02:34 +01002541 intrinsic_slow_path,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002542 temp1,
donghui.baic2ec9ad2016-03-10 14:02:55 +08002543 optimizations.GetCountIsDestinationLength());
2544 {
2545 // We use a block to end the scratch scope before the write barrier, thus
2546 // freeing the temporary registers so they can be used in `MarkGCCard`.
2547 UseScratchRegisterScope temps(masm);
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002548 Location temp3_loc; // Used only for Baker read barrier.
Roland Levillain54f869e2017-03-06 13:54:11 +00002549 Register temp3;
2550 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002551 temp3_loc = locations->GetTemp(2);
2552 temp3 = WRegisterFrom(temp3_loc);
Roland Levillain54f869e2017-03-06 13:54:11 +00002553 } else {
2554 temp3 = temps.AcquireW();
2555 }
Roland Levillain0b671c02016-08-19 12:02:34 +01002556
donghui.baic2ec9ad2016-03-10 14:02:55 +08002557 if (!optimizations.GetDoesNotNeedTypeCheck()) {
2558 // Check whether all elements of the source array are assignable to the component
2559 // type of the destination array. We do two checks: the classes are the same,
2560 // or the destination is Object[]. If none of these checks succeed, we go to the
2561 // slow path.
donghui.baic2ec9ad2016-03-10 14:02:55 +08002562
Roland Levillain0b671c02016-08-19 12:02:34 +01002563 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2564 if (!optimizations.GetSourceIsNonPrimitiveArray()) {
2565 // /* HeapReference<Class> */ temp1 = src->klass_
2566 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2567 temp1_loc,
2568 src.W(),
2569 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002570 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002571 /* needs_null_check */ false,
2572 /* use_load_acquire */ false);
2573 // Bail out if the source is not a non primitive array.
2574 // /* HeapReference<Class> */ temp1 = temp1->component_type_
2575 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2576 temp1_loc,
2577 temp1,
2578 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002579 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002580 /* needs_null_check */ false,
2581 /* use_load_acquire */ false);
2582 __ Cbz(temp1, intrinsic_slow_path->GetEntryLabel());
2583 // If heap poisoning is enabled, `temp1` has been unpoisoned
2584 // by the the previous call to GenerateFieldLoadWithBakerReadBarrier.
2585 // /* uint16_t */ temp1 = static_cast<uint16>(temp1->primitive_type_);
2586 __ Ldrh(temp1, HeapOperand(temp1, primitive_offset));
2587 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2588 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002589 }
Roland Levillain0b671c02016-08-19 12:02:34 +01002590
2591 // /* HeapReference<Class> */ temp1 = dest->klass_
2592 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2593 temp1_loc,
2594 dest.W(),
2595 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002596 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002597 /* needs_null_check */ false,
2598 /* use_load_acquire */ false);
2599
2600 if (!optimizations.GetDestinationIsNonPrimitiveArray()) {
2601 // Bail out if the destination is not a non primitive array.
2602 //
2603 // Register `temp1` is not trashed by the read barrier emitted
2604 // by GenerateFieldLoadWithBakerReadBarrier below, as that
2605 // method produces a call to a ReadBarrierMarkRegX entry point,
2606 // which saves all potentially live registers, including
2607 // temporaries such a `temp1`.
2608 // /* HeapReference<Class> */ temp2 = temp1->component_type_
2609 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2610 temp2_loc,
2611 temp1,
2612 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002613 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002614 /* needs_null_check */ false,
2615 /* use_load_acquire */ false);
2616 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel());
2617 // If heap poisoning is enabled, `temp2` has been unpoisoned
2618 // by the the previous call to GenerateFieldLoadWithBakerReadBarrier.
2619 // /* uint16_t */ temp2 = static_cast<uint16>(temp2->primitive_type_);
2620 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset));
2621 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2622 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel());
2623 }
2624
2625 // For the same reason given earlier, `temp1` is not trashed by the
2626 // read barrier emitted by GenerateFieldLoadWithBakerReadBarrier below.
2627 // /* HeapReference<Class> */ temp2 = src->klass_
2628 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2629 temp2_loc,
2630 src.W(),
2631 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002632 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002633 /* needs_null_check */ false,
2634 /* use_load_acquire */ false);
2635 // Note: if heap poisoning is on, we are comparing two unpoisoned references here.
2636 __ Cmp(temp1, temp2);
2637
2638 if (optimizations.GetDestinationIsTypedObjectArray()) {
2639 vixl::aarch64::Label do_copy;
2640 __ B(&do_copy, eq);
2641 // /* HeapReference<Class> */ temp1 = temp1->component_type_
2642 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2643 temp1_loc,
2644 temp1,
2645 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002646 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002647 /* needs_null_check */ false,
2648 /* use_load_acquire */ false);
2649 // /* HeapReference<Class> */ temp1 = temp1->super_class_
2650 // We do not need to emit a read barrier for the following
2651 // heap reference load, as `temp1` is only used in a
2652 // comparison with null below, and this reference is not
2653 // kept afterwards.
2654 __ Ldr(temp1, HeapOperand(temp1, super_offset));
2655 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel());
2656 __ Bind(&do_copy);
2657 } else {
2658 __ B(intrinsic_slow_path->GetEntryLabel(), ne);
2659 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002660 } else {
Roland Levillain0b671c02016-08-19 12:02:34 +01002661 // Non read barrier code.
2662
2663 // /* HeapReference<Class> */ temp1 = dest->klass_
2664 __ Ldr(temp1, MemOperand(dest, class_offset));
2665 // /* HeapReference<Class> */ temp2 = src->klass_
2666 __ Ldr(temp2, MemOperand(src, class_offset));
2667 bool did_unpoison = false;
2668 if (!optimizations.GetDestinationIsNonPrimitiveArray() ||
2669 !optimizations.GetSourceIsNonPrimitiveArray()) {
2670 // One or two of the references need to be unpoisoned. Unpoison them
2671 // both to make the identity check valid.
2672 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2673 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
2674 did_unpoison = true;
2675 }
2676
2677 if (!optimizations.GetDestinationIsNonPrimitiveArray()) {
2678 // Bail out if the destination is not a non primitive array.
2679 // /* HeapReference<Class> */ temp3 = temp1->component_type_
2680 __ Ldr(temp3, HeapOperand(temp1, component_offset));
2681 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel());
2682 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp3);
2683 // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_);
2684 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset));
2685 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2686 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel());
2687 }
2688
2689 if (!optimizations.GetSourceIsNonPrimitiveArray()) {
2690 // Bail out if the source is not a non primitive array.
2691 // /* HeapReference<Class> */ temp3 = temp2->component_type_
2692 __ Ldr(temp3, HeapOperand(temp2, component_offset));
2693 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel());
2694 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp3);
2695 // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_);
2696 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset));
2697 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2698 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel());
2699 }
2700
2701 __ Cmp(temp1, temp2);
2702
2703 if (optimizations.GetDestinationIsTypedObjectArray()) {
2704 vixl::aarch64::Label do_copy;
2705 __ B(&do_copy, eq);
2706 if (!did_unpoison) {
2707 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2708 }
2709 // /* HeapReference<Class> */ temp1 = temp1->component_type_
2710 __ Ldr(temp1, HeapOperand(temp1, component_offset));
2711 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2712 // /* HeapReference<Class> */ temp1 = temp1->super_class_
2713 __ Ldr(temp1, HeapOperand(temp1, super_offset));
2714 // No need to unpoison the result, we're comparing against null.
2715 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel());
2716 __ Bind(&do_copy);
2717 } else {
2718 __ B(intrinsic_slow_path->GetEntryLabel(), ne);
2719 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002720 }
2721 } else if (!optimizations.GetSourceIsNonPrimitiveArray()) {
2722 DCHECK(optimizations.GetDestinationIsNonPrimitiveArray());
2723 // Bail out if the source is not a non primitive array.
Roland Levillain0b671c02016-08-19 12:02:34 +01002724 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2725 // /* HeapReference<Class> */ temp1 = src->klass_
2726 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2727 temp1_loc,
2728 src.W(),
2729 class_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002730 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002731 /* needs_null_check */ false,
2732 /* use_load_acquire */ false);
2733 // /* HeapReference<Class> */ temp2 = temp1->component_type_
2734 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
2735 temp2_loc,
2736 temp1,
2737 component_offset,
Vladimir Markof4f2daa2017-03-20 18:26:59 +00002738 temp3_loc,
Roland Levillain0b671c02016-08-19 12:02:34 +01002739 /* needs_null_check */ false,
2740 /* use_load_acquire */ false);
2741 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel());
2742 // If heap poisoning is enabled, `temp2` has been unpoisoned
2743 // by the the previous call to GenerateFieldLoadWithBakerReadBarrier.
2744 } else {
2745 // /* HeapReference<Class> */ temp1 = src->klass_
2746 __ Ldr(temp1, HeapOperand(src.W(), class_offset));
2747 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp1);
2748 // /* HeapReference<Class> */ temp2 = temp1->component_type_
2749 __ Ldr(temp2, HeapOperand(temp1, component_offset));
2750 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel());
2751 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(temp2);
2752 }
2753 // /* uint16_t */ temp2 = static_cast<uint16>(temp2->primitive_type_);
2754 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset));
donghui.baic2ec9ad2016-03-10 14:02:55 +08002755 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
Roland Levillain0b671c02016-08-19 12:02:34 +01002756 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002757 }
2758
Roland Levillain1663d162017-03-17 15:15:21 +00002759 if (length.IsConstant() && length.GetConstant()->AsIntConstant()->GetValue() == 0) {
2760 // Null constant length: not need to emit the loop code at all.
Roland Levillain0b671c02016-08-19 12:02:34 +01002761 } else {
Roland Levillain1663d162017-03-17 15:15:21 +00002762 Register src_curr_addr = temp1.X();
2763 Register dst_curr_addr = temp2.X();
2764 Register src_stop_addr = temp3.X();
2765 vixl::aarch64::Label done;
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002766 const DataType::Type type = DataType::Type::kReference;
2767 const int32_t element_size = DataType::Size(type);
Roland Levillain1663d162017-03-17 15:15:21 +00002768
2769 if (length.IsRegister()) {
2770 // Don't enter the copy loop if the length is null.
2771 __ Cbz(WRegisterFrom(length), &done);
2772 }
2773
2774 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
2775 // TODO: Also convert this intrinsic to the IsGcMarking strategy?
2776
2777 // SystemArrayCopy implementation for Baker read barriers (see
Roland Levillain9983e302017-07-14 14:34:22 +01002778 // also CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier):
Roland Levillain1663d162017-03-17 15:15:21 +00002779 //
2780 // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState();
2781 // lfence; // Load fence or artificial data dependency to prevent load-load reordering
2782 // bool is_gray = (rb_state == ReadBarrier::GrayState());
2783 // if (is_gray) {
2784 // // Slow-path copy.
2785 // do {
2786 // *dest_ptr++ = MaybePoison(ReadBarrier::Mark(MaybeUnpoison(*src_ptr++)));
2787 // } while (src_ptr != end_ptr)
2788 // } else {
2789 // // Fast-path copy.
2790 // do {
2791 // *dest_ptr++ = *src_ptr++;
2792 // } while (src_ptr != end_ptr)
2793 // }
2794
2795 // Make sure `tmp` is not IP0, as it is clobbered by
2796 // ReadBarrierMarkRegX entry points in
2797 // ReadBarrierSystemArrayCopySlowPathARM64.
Roland Levillain1ca955d2017-04-13 19:34:30 +01002798 DCHECK(temps.IsAvailable(ip0));
Roland Levillain1663d162017-03-17 15:15:21 +00002799 temps.Exclude(ip0);
Roland Levillain0b671c02016-08-19 12:02:34 +01002800 Register tmp = temps.AcquireW();
Roland Levillain1663d162017-03-17 15:15:21 +00002801 DCHECK_NE(LocationFrom(tmp).reg(), IP0);
Roland Levillain1ca955d2017-04-13 19:34:30 +01002802 // Put IP0 back in the pool so that VIXL has at least one
2803 // scratch register available to emit macro-instructions (note
2804 // that IP1 is already used for `tmp`). Indeed some
2805 // macro-instructions used in GenSystemArrayCopyAddresses
2806 // (invoked hereunder) may require a scratch register (for
2807 // instance to emit a load with a large constant offset).
2808 temps.Include(ip0);
Roland Levillain1663d162017-03-17 15:15:21 +00002809
2810 // /* int32_t */ monitor = src->monitor_
2811 __ Ldr(tmp, HeapOperand(src.W(), monitor_offset));
2812 // /* LockWord */ lock_word = LockWord(monitor)
2813 static_assert(sizeof(LockWord) == sizeof(int32_t),
2814 "art::LockWord and int32_t have different sizes.");
2815
2816 // Introduce a dependency on the lock_word including rb_state,
2817 // to prevent load-load reordering, and without using
2818 // a memory barrier (which would be more expensive).
2819 // `src` is unchanged by this operation, but its value now depends
2820 // on `tmp`.
2821 __ Add(src.X(), src.X(), Operand(tmp.X(), LSR, 32));
2822
2823 // Compute base source address, base destination address, and end
2824 // source address for System.arraycopy* intrinsics in `src_base`,
2825 // `dst_base` and `src_end` respectively.
2826 // Note that `src_curr_addr` is computed from from `src` (and
2827 // `src_pos`) here, and thus honors the artificial dependency
2828 // of `src` on `tmp`.
2829 GenSystemArrayCopyAddresses(masm,
2830 type,
2831 src,
2832 src_pos,
2833 dest,
2834 dest_pos,
2835 length,
2836 src_curr_addr,
2837 dst_curr_addr,
2838 src_stop_addr);
2839
2840 // Slow path used to copy array when `src` is gray.
2841 SlowPathCodeARM64* read_barrier_slow_path =
2842 new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathARM64(invoke, LocationFrom(tmp));
2843 codegen_->AddSlowPath(read_barrier_slow_path);
2844
2845 // Given the numeric representation, it's enough to check the low bit of the rb_state.
2846 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
2847 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
2848 __ Tbnz(tmp, LockWord::kReadBarrierStateShift, read_barrier_slow_path->GetEntryLabel());
2849
2850 // Fast-path copy.
2851 // Iterate over the arrays and do a raw copy of the objects. We don't need to
2852 // poison/unpoison.
2853 vixl::aarch64::Label loop;
2854 __ Bind(&loop);
Roland Levillain0b671c02016-08-19 12:02:34 +01002855 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex));
2856 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex));
Roland Levillain1663d162017-03-17 15:15:21 +00002857 __ Cmp(src_curr_addr, src_stop_addr);
2858 __ B(&loop, ne);
2859
2860 __ Bind(read_barrier_slow_path->GetExitLabel());
2861 } else {
2862 // Non read barrier code.
2863 // Compute base source address, base destination address, and end
2864 // source address for System.arraycopy* intrinsics in `src_base`,
2865 // `dst_base` and `src_end` respectively.
2866 GenSystemArrayCopyAddresses(masm,
2867 type,
2868 src,
2869 src_pos,
2870 dest,
2871 dest_pos,
2872 length,
2873 src_curr_addr,
2874 dst_curr_addr,
2875 src_stop_addr);
2876 // Iterate over the arrays and do a raw copy of the objects. We don't need to
2877 // poison/unpoison.
2878 vixl::aarch64::Label loop;
2879 __ Bind(&loop);
2880 {
2881 Register tmp = temps.AcquireW();
2882 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex));
2883 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex));
2884 }
2885 __ Cmp(src_curr_addr, src_stop_addr);
2886 __ B(&loop, ne);
Roland Levillain0b671c02016-08-19 12:02:34 +01002887 }
Roland Levillain0b671c02016-08-19 12:02:34 +01002888 __ Bind(&done);
donghui.baic2ec9ad2016-03-10 14:02:55 +08002889 }
donghui.baic2ec9ad2016-03-10 14:02:55 +08002890 }
Roland Levillain9cc0ea82017-03-16 11:25:59 +00002891
donghui.baic2ec9ad2016-03-10 14:02:55 +08002892 // We only need one card marking on the destination array.
2893 codegen_->MarkGCCard(dest.W(), Register(), /* value_can_be_null */ false);
2894
Roland Levillain0b671c02016-08-19 12:02:34 +01002895 __ Bind(intrinsic_slow_path->GetExitLabel());
donghui.baic2ec9ad2016-03-10 14:02:55 +08002896}
2897
Anton Kirilova3ffea22016-04-07 17:02:37 +01002898static void GenIsInfinite(LocationSummary* locations,
2899 bool is64bit,
Scott Wakeling97c72b72016-06-24 16:19:36 +01002900 MacroAssembler* masm) {
Anton Kirilova3ffea22016-04-07 17:02:37 +01002901 Operand infinity;
2902 Register out;
2903
2904 if (is64bit) {
2905 infinity = kPositiveInfinityDouble;
2906 out = XRegisterFrom(locations->Out());
2907 } else {
2908 infinity = kPositiveInfinityFloat;
2909 out = WRegisterFrom(locations->Out());
2910 }
2911
Scott Wakeling97c72b72016-06-24 16:19:36 +01002912 const Register zero = vixl::aarch64::Assembler::AppropriateZeroRegFor(out);
Anton Kirilova3ffea22016-04-07 17:02:37 +01002913
2914 MoveFPToInt(locations, is64bit, masm);
2915 __ Eor(out, out, infinity);
2916 // We don't care about the sign bit, so shift left.
2917 __ Cmp(zero, Operand(out, LSL, 1));
2918 __ Cset(out, eq);
2919}
2920
2921void IntrinsicLocationsBuilderARM64::VisitFloatIsInfinite(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01002922 CreateFPToIntLocations(allocator_, invoke);
Anton Kirilova3ffea22016-04-07 17:02:37 +01002923}
2924
2925void IntrinsicCodeGeneratorARM64::VisitFloatIsInfinite(HInvoke* invoke) {
2926 GenIsInfinite(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
2927}
2928
2929void IntrinsicLocationsBuilderARM64::VisitDoubleIsInfinite(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01002930 CreateFPToIntLocations(allocator_, invoke);
Anton Kirilova3ffea22016-04-07 17:02:37 +01002931}
2932
2933void IntrinsicCodeGeneratorARM64::VisitDoubleIsInfinite(HInvoke* invoke) {
2934 GenIsInfinite(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
2935}
2936
Nicolas Geoffray331605a2017-03-01 11:01:41 +00002937void IntrinsicLocationsBuilderARM64::VisitIntegerValueOf(HInvoke* invoke) {
2938 InvokeRuntimeCallingConvention calling_convention;
2939 IntrinsicVisitor::ComputeIntegerValueOfLocations(
2940 invoke,
2941 codegen_,
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002942 calling_convention.GetReturnLocation(DataType::Type::kReference),
Nicolas Geoffray331605a2017-03-01 11:01:41 +00002943 Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode()));
2944}
2945
2946void IntrinsicCodeGeneratorARM64::VisitIntegerValueOf(HInvoke* invoke) {
2947 IntrinsicVisitor::IntegerValueOfInfo info = IntrinsicVisitor::ComputeIntegerValueOfInfo();
2948 LocationSummary* locations = invoke->GetLocations();
2949 MacroAssembler* masm = GetVIXLAssembler();
2950
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002951 Register out = RegisterFrom(locations->Out(), DataType::Type::kReference);
Nicolas Geoffray331605a2017-03-01 11:01:41 +00002952 UseScratchRegisterScope temps(masm);
2953 Register temp = temps.AcquireW();
2954 InvokeRuntimeCallingConvention calling_convention;
2955 Register argument = calling_convention.GetRegisterAt(0);
2956 if (invoke->InputAt(0)->IsConstant()) {
2957 int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue();
2958 if (value >= info.low && value <= info.high) {
2959 // Just embed the j.l.Integer in the code.
2960 ScopedObjectAccess soa(Thread::Current());
2961 mirror::Object* boxed = info.cache->Get(value + (-info.low));
2962 DCHECK(boxed != nullptr && Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(boxed));
2963 uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(boxed));
2964 __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(address));
2965 } else {
2966 // Allocate and initialize a new j.l.Integer.
2967 // TODO: If we JIT, we could allocate the j.l.Integer now, and store it in the
2968 // JIT object table.
2969 uint32_t address =
2970 dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer));
2971 __ Ldr(argument.W(), codegen_->DeduplicateBootImageAddressLiteral(address));
2972 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc());
2973 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
2974 __ Mov(temp.W(), value);
2975 __ Str(temp.W(), HeapOperand(out.W(), info.value_offset));
2976 // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation
2977 // one.
2978 codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore);
2979 }
2980 } else {
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002981 Register in = RegisterFrom(locations->InAt(0), DataType::Type::kInt32);
Nicolas Geoffray331605a2017-03-01 11:01:41 +00002982 // Check bounds of our cache.
2983 __ Add(out.W(), in.W(), -info.low);
2984 __ Cmp(out.W(), info.high - info.low + 1);
2985 vixl::aarch64::Label allocate, done;
2986 __ B(&allocate, hs);
2987 // If the value is within the bounds, load the j.l.Integer directly from the array.
2988 uint32_t data_offset = mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value();
2989 uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.cache));
2990 __ Ldr(temp.W(), codegen_->DeduplicateBootImageAddressLiteral(data_offset + address));
2991 MemOperand source = HeapOperand(
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01002992 temp, out.X(), LSL, DataType::SizeShift(DataType::Type::kReference));
2993 codegen_->Load(DataType::Type::kReference, out, source);
Nicolas Geoffray331605a2017-03-01 11:01:41 +00002994 codegen_->GetAssembler()->MaybeUnpoisonHeapReference(out);
2995 __ B(&done);
2996 __ Bind(&allocate);
2997 // Otherwise allocate and initialize a new j.l.Integer.
2998 address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer));
2999 __ Ldr(argument.W(), codegen_->DeduplicateBootImageAddressLiteral(address));
3000 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc());
3001 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
3002 __ Str(in.W(), HeapOperand(out.W(), info.value_offset));
3003 // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation
3004 // one.
3005 codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore);
3006 __ Bind(&done);
3007 }
3008}
3009
Nicolas Geoffray365719c2017-03-08 13:11:50 +00003010void IntrinsicLocationsBuilderARM64::VisitThreadInterrupted(HInvoke* invoke) {
Vladimir Markoca6fff82017-10-03 14:49:14 +01003011 LocationSummary* locations =
3012 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
Nicolas Geoffray365719c2017-03-08 13:11:50 +00003013 locations->SetOut(Location::RequiresRegister());
3014}
3015
3016void IntrinsicCodeGeneratorARM64::VisitThreadInterrupted(HInvoke* invoke) {
3017 MacroAssembler* masm = GetVIXLAssembler();
Vladimir Marko0ebe0d82017-09-21 22:50:39 +01003018 Register out = RegisterFrom(invoke->GetLocations()->Out(), DataType::Type::kInt32);
Nicolas Geoffray365719c2017-03-08 13:11:50 +00003019 UseScratchRegisterScope temps(masm);
3020 Register temp = temps.AcquireX();
3021
3022 __ Add(temp, tr, Thread::InterruptedOffset<kArm64PointerSize>().Int32Value());
3023 __ Ldar(out.W(), MemOperand(temp));
3024
3025 vixl::aarch64::Label done;
3026 __ Cbz(out.W(), &done);
3027 __ Stlr(wzr, MemOperand(temp));
3028 __ Bind(&done);
3029}
3030
Vladimir Markod254f5c2017-06-02 15:18:36 +00003031UNIMPLEMENTED_INTRINSIC(ARM64, ReferenceGetReferent)
Andreas Gampe878d58c2015-01-15 23:24:00 -08003032
Aart Bikff7d89c2016-11-07 08:49:28 -08003033UNIMPLEMENTED_INTRINSIC(ARM64, StringStringIndexOf);
3034UNIMPLEMENTED_INTRINSIC(ARM64, StringStringIndexOfAfter);
Aart Bik71bf7b42016-11-16 10:17:46 -08003035UNIMPLEMENTED_INTRINSIC(ARM64, StringBufferAppend);
3036UNIMPLEMENTED_INTRINSIC(ARM64, StringBufferLength);
3037UNIMPLEMENTED_INTRINSIC(ARM64, StringBufferToString);
3038UNIMPLEMENTED_INTRINSIC(ARM64, StringBuilderAppend);
3039UNIMPLEMENTED_INTRINSIC(ARM64, StringBuilderLength);
3040UNIMPLEMENTED_INTRINSIC(ARM64, StringBuilderToString);
Aart Bikff7d89c2016-11-07 08:49:28 -08003041
Aart Bik0e54c012016-03-04 12:08:31 -08003042// 1.8.
3043UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndAddInt)
3044UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndAddLong)
3045UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetInt)
3046UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetLong)
3047UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetObject)
Aart Bik0e54c012016-03-04 12:08:31 -08003048
Aart Bik2f9fcc92016-03-01 15:16:54 -08003049UNREACHABLE_INTRINSICS(ARM64)
Roland Levillain4d027112015-07-01 15:41:14 +01003050
3051#undef __
3052
Andreas Gampe878d58c2015-01-15 23:24:00 -08003053} // namespace arm64
3054} // namespace art