Optimizing: Add direct calls to math intrinsics
Support the double forms of:
cos, sin, acos, asin, atan, atan2, cbrt, cosh, exp, expm1,
hypot, log, log10, nextAfter, sinh, tan, tanh
Add these entries to the vector addressed off the thread pointer. Call
the libc routines directly, which means that we have to implement the
native ABI, not the ART one. For x86_64, that includes saving XMM12-15
as the native ABI considers them caller-save, while the ART ABI
considers them callee-save. We save them by marking them as used by the
call to the math function. For x86, this is not an issue, as all the XMM
registers are caller-save.
Other architectures will call Java as before until they are ready to
implement the new intrinsics.
Bump the OAT version since we are incompatible with old boot.oat files.
Change-Id: Ic6332c3555c09393a17d1ad4daf62932488722fb
Signed-off-by: Mark Mendell <mark.p.mendell@intel.com>
diff --git a/compiler/dex/quick/dex_file_method_inliner.cc b/compiler/dex/quick/dex_file_method_inliner.cc
index eaf2408..f48947d 100644
--- a/compiler/dex/quick/dex_file_method_inliner.cc
+++ b/compiler/dex/quick/dex_file_method_inliner.cc
@@ -50,6 +50,23 @@
true, // kIntrinsicMinMaxLong
true, // kIntrinsicMinMaxFloat
true, // kIntrinsicMinMaxDouble
+ true, // kIntrinsicCos
+ true, // kIntrinsicSin
+ true, // kIntrinsicAcos
+ true, // kIntrinsicAsin
+ true, // kIntrinsicAtan
+ true, // kIntrinsicAtan2
+ true, // kIntrinsicCbrt
+ true, // kIntrinsicCosh
+ true, // kIntrinsicExp
+ true, // kIntrinsicExpm1
+ true, // kIntrinsicHypot
+ true, // kIntrinsicLog
+ true, // kIntrinsicLog10
+ true, // kIntrinsicNextAfter
+ true, // kIntrinsicSinh
+ true, // kIntrinsicTan
+ true, // kIntrinsicTanh
true, // kIntrinsicSqrt
true, // kIntrinsicCeil
true, // kIntrinsicFloor
@@ -95,6 +112,23 @@
static_assert(kIntrinsicIsStatic[kIntrinsicMinMaxLong], "MinMaxLong_must_be_static");
static_assert(kIntrinsicIsStatic[kIntrinsicMinMaxFloat], "MinMaxFloat_must_be_static");
static_assert(kIntrinsicIsStatic[kIntrinsicMinMaxDouble], "MinMaxDouble_must_be_static");
+static_assert(kIntrinsicIsStatic[kIntrinsicCos], "Cos must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicSin], "Sin must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicAcos], "Acos must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicAsin], "Asin must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicAtan], "Atan must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicAtan2], "Atan2 must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicCbrt], "Cbrt must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicCosh], "Cosh must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicExp], "Exp must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicExpm1], "Expm1 must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicHypot], "Hypot must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicLog], "Log must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicLog10], "Log10 must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicNextAfter], "NextAfter must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicSinh], "Sinh must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicTan], "Tan must be static");
+static_assert(kIntrinsicIsStatic[kIntrinsicTanh], "Tanh must be static");
static_assert(kIntrinsicIsStatic[kIntrinsicSqrt], "Sqrt must be static");
static_assert(kIntrinsicIsStatic[kIntrinsicCeil], "Ceil must be static");
static_assert(kIntrinsicIsStatic[kIntrinsicFloor], "Floor must be static");
@@ -196,6 +230,23 @@
"abs", // kNameCacheAbs
"max", // kNameCacheMax
"min", // kNameCacheMin
+ "cos", // kNameCacheCos
+ "sin", // kNameCacheSin
+ "acos", // kNameCacheAcos
+ "asin", // kNameCacheAsin
+ "atan", // kNameCacheAtan
+ "atan2", // kNameCacheAtan2
+ "cbrt", // kNameCacheCbrt
+ "cosh", // kNameCacheCosh
+ "exp", // kNameCacheExp
+ "expm1", // kNameCacheExpm1
+ "hypot", // kNameCacheHypot
+ "log", // kNameCacheLog
+ "log10", // kNameCacheLog10
+ "nextAfter", // kNameCacheNextAfter
+ "sinh", // kNameCacheSinh
+ "tan", // kNameCacheTan
+ "tanh", // kNameCacheTanh
"sqrt", // kNameCacheSqrt
"ceil", // kNameCacheCeil
"floor", // kNameCacheFloor
@@ -425,6 +476,23 @@
INTRINSIC(JavaLangMath, Max, DD_D, kIntrinsicMinMaxDouble, kIntrinsicFlagMax),
INTRINSIC(JavaLangStrictMath, Max, DD_D, kIntrinsicMinMaxDouble, kIntrinsicFlagMax),
+ INTRINSIC(JavaLangMath, Cos, D_D, kIntrinsicCos, 0),
+ INTRINSIC(JavaLangMath, Sin, D_D, kIntrinsicSin, 0),
+ INTRINSIC(JavaLangMath, Acos, D_D, kIntrinsicAcos, 0),
+ INTRINSIC(JavaLangMath, Asin, D_D, kIntrinsicAsin, 0),
+ INTRINSIC(JavaLangMath, Atan, D_D, kIntrinsicAtan, 0),
+ INTRINSIC(JavaLangMath, Atan2, DD_D, kIntrinsicAtan2, 0),
+ INTRINSIC(JavaLangMath, Cbrt, D_D, kIntrinsicCbrt, 0),
+ INTRINSIC(JavaLangMath, Cosh, D_D, kIntrinsicCosh, 0),
+ INTRINSIC(JavaLangMath, Exp, D_D, kIntrinsicExp, 0),
+ INTRINSIC(JavaLangMath, Expm1, D_D, kIntrinsicExpm1, 0),
+ INTRINSIC(JavaLangMath, Hypot, DD_D, kIntrinsicHypot, 0),
+ INTRINSIC(JavaLangMath, Log, D_D, kIntrinsicLog, 0),
+ INTRINSIC(JavaLangMath, Log10, D_D, kIntrinsicLog10, 0),
+ INTRINSIC(JavaLangMath, NextAfter, DD_D, kIntrinsicNextAfter, 0),
+ INTRINSIC(JavaLangMath, Sinh, D_D, kIntrinsicSinh, 0),
+ INTRINSIC(JavaLangMath, Tan, D_D, kIntrinsicTan, 0),
+ INTRINSIC(JavaLangMath, Tanh, D_D, kIntrinsicTanh, 0),
INTRINSIC(JavaLangMath, Sqrt, D_D, kIntrinsicSqrt, 0),
INTRINSIC(JavaLangStrictMath, Sqrt, D_D, kIntrinsicSqrt, 0),
@@ -603,6 +671,25 @@
return backend->GenInlinedMinMaxFP(info, intrinsic.d.data & kIntrinsicFlagMin, false /* is_double */);
case kIntrinsicMinMaxDouble:
return backend->GenInlinedMinMaxFP(info, intrinsic.d.data & kIntrinsicFlagMin, true /* is_double */);
+ case kIntrinsicCos:
+ case kIntrinsicSin:
+ case kIntrinsicAcos:
+ case kIntrinsicAsin:
+ case kIntrinsicAtan:
+ case kIntrinsicAtan2:
+ case kIntrinsicCbrt:
+ case kIntrinsicCosh:
+ case kIntrinsicExp:
+ case kIntrinsicExpm1:
+ case kIntrinsicHypot:
+ case kIntrinsicLog:
+ case kIntrinsicLog10:
+ case kIntrinsicNextAfter:
+ case kIntrinsicSinh:
+ case kIntrinsicTan:
+ case kIntrinsicTanh:
+ // Not implemented in Quick.
+ return false;
case kIntrinsicSqrt:
return backend->GenInlinedSqrt(info);
case kIntrinsicCeil:
diff --git a/compiler/dex/quick/dex_file_method_inliner.h b/compiler/dex/quick/dex_file_method_inliner.h
index 5ce110c..ac70577 100644
--- a/compiler/dex/quick/dex_file_method_inliner.h
+++ b/compiler/dex/quick/dex_file_method_inliner.h
@@ -162,6 +162,23 @@
kNameCacheAbs,
kNameCacheMax,
kNameCacheMin,
+ kNameCacheCos,
+ kNameCacheSin,
+ kNameCacheAcos,
+ kNameCacheAsin,
+ kNameCacheAtan,
+ kNameCacheAtan2,
+ kNameCacheCbrt,
+ kNameCacheCosh,
+ kNameCacheExp,
+ kNameCacheExpm1,
+ kNameCacheHypot,
+ kNameCacheLog,
+ kNameCacheLog10,
+ kNameCacheNextAfter,
+ kNameCacheSinh,
+ kNameCacheTan,
+ kNameCacheTanh,
kNameCacheSqrt,
kNameCacheCeil,
kNameCacheFloor,
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc
index b8610d0..8ff0133 100644
--- a/compiler/oat_test.cc
+++ b/compiler/oat_test.cc
@@ -260,7 +260,7 @@
EXPECT_EQ(72U, sizeof(OatHeader));
EXPECT_EQ(4U, sizeof(OatMethodOffsets));
EXPECT_EQ(28U, sizeof(OatQuickMethodHeader));
- EXPECT_EQ(114 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints));
+ EXPECT_EQ(131 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints));
}
TEST_F(OatTest, OatHeaderIsValid) {
diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h
index 145b1f3..7351fed 100644
--- a/compiler/optimizing/code_generator_x86_64.h
+++ b/compiler/optimizing/code_generator_x86_64.h
@@ -47,6 +47,12 @@
static constexpr size_t kRuntimeParameterFpuRegistersLength =
arraysize(kRuntimeParameterFpuRegisters);
+// These XMM registers are non-volatile in ART ABI, but volatile in native ABI.
+// If the ART ABI changes, this list must be updated. It is used to ensure that
+// these are not clobbered by any direct call to native code (such as math intrinsics).
+static constexpr FloatRegister non_volatile_xmm_regs[] = { XMM12, XMM13, XMM14, XMM15 };
+
+
class InvokeRuntimeCallingConvention : public CallingConvention<Register, FloatRegister> {
public:
InvokeRuntimeCallingConvention()
diff --git a/compiler/optimizing/intrinsics.cc b/compiler/optimizing/intrinsics.cc
index 8340811..7127215 100644
--- a/compiler/optimizing/intrinsics.cc
+++ b/compiler/optimizing/intrinsics.cc
@@ -189,6 +189,42 @@
return ((method.d.data & kIntrinsicFlagMin) == 0) ?
Intrinsics::kMathMaxLongLong : Intrinsics::kMathMinLongLong;
+ // More math builtins.
+ case kIntrinsicCos:
+ return Intrinsics::kMathCos;
+ case kIntrinsicSin:
+ return Intrinsics::kMathSin;
+ case kIntrinsicAcos:
+ return Intrinsics::kMathAcos;
+ case kIntrinsicAsin:
+ return Intrinsics::kMathAsin;
+ case kIntrinsicAtan:
+ return Intrinsics::kMathAtan;
+ case kIntrinsicAtan2:
+ return Intrinsics::kMathAtan2;
+ case kIntrinsicCbrt:
+ return Intrinsics::kMathCbrt;
+ case kIntrinsicCosh:
+ return Intrinsics::kMathCosh;
+ case kIntrinsicExp:
+ return Intrinsics::kMathExp;
+ case kIntrinsicExpm1:
+ return Intrinsics::kMathExpm1;
+ case kIntrinsicHypot:
+ return Intrinsics::kMathHypot;
+ case kIntrinsicLog:
+ return Intrinsics::kMathLog;
+ case kIntrinsicLog10:
+ return Intrinsics::kMathLog10;
+ case kIntrinsicNextAfter:
+ return Intrinsics::kMathNextAfter;
+ case kIntrinsicSinh:
+ return Intrinsics::kMathSinh;
+ case kIntrinsicTan:
+ return Intrinsics::kMathTan;
+ case kIntrinsicTanh:
+ return Intrinsics::kMathTanh;
+
// Misc math.
case kIntrinsicSqrt:
return Intrinsics::kMathSqrt;
diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc
index 5329b5c..9820c5d 100644
--- a/compiler/optimizing/intrinsics_arm.cc
+++ b/compiler/optimizing/intrinsics_arm.cc
@@ -1718,6 +1718,23 @@
UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
+UNIMPLEMENTED_INTRINSIC(MathCos)
+UNIMPLEMENTED_INTRINSIC(MathSin)
+UNIMPLEMENTED_INTRINSIC(MathAcos)
+UNIMPLEMENTED_INTRINSIC(MathAsin)
+UNIMPLEMENTED_INTRINSIC(MathAtan)
+UNIMPLEMENTED_INTRINSIC(MathAtan2)
+UNIMPLEMENTED_INTRINSIC(MathCbrt)
+UNIMPLEMENTED_INTRINSIC(MathCosh)
+UNIMPLEMENTED_INTRINSIC(MathExp)
+UNIMPLEMENTED_INTRINSIC(MathExpm1)
+UNIMPLEMENTED_INTRINSIC(MathHypot)
+UNIMPLEMENTED_INTRINSIC(MathLog)
+UNIMPLEMENTED_INTRINSIC(MathLog10)
+UNIMPLEMENTED_INTRINSIC(MathNextAfter)
+UNIMPLEMENTED_INTRINSIC(MathSinh)
+UNIMPLEMENTED_INTRINSIC(MathTan)
+UNIMPLEMENTED_INTRINSIC(MathTanh)
#undef UNIMPLEMENTED_INTRINSIC
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index 962c4d5..2e5b022 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -1532,6 +1532,24 @@
UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
+UNIMPLEMENTED_INTRINSIC(MathCos)
+UNIMPLEMENTED_INTRINSIC(MathSin)
+UNIMPLEMENTED_INTRINSIC(MathAcos)
+UNIMPLEMENTED_INTRINSIC(MathAsin)
+UNIMPLEMENTED_INTRINSIC(MathAtan)
+UNIMPLEMENTED_INTRINSIC(MathAtan2)
+UNIMPLEMENTED_INTRINSIC(MathCbrt)
+UNIMPLEMENTED_INTRINSIC(MathCosh)
+UNIMPLEMENTED_INTRINSIC(MathExp)
+UNIMPLEMENTED_INTRINSIC(MathExpm1)
+UNIMPLEMENTED_INTRINSIC(MathHypot)
+UNIMPLEMENTED_INTRINSIC(MathLog)
+UNIMPLEMENTED_INTRINSIC(MathLog10)
+UNIMPLEMENTED_INTRINSIC(MathNextAfter)
+UNIMPLEMENTED_INTRINSIC(MathSinh)
+UNIMPLEMENTED_INTRINSIC(MathTan)
+UNIMPLEMENTED_INTRINSIC(MathTanh)
+
#undef UNIMPLEMENTED_INTRINSIC
#undef __
diff --git a/compiler/optimizing/intrinsics_list.h b/compiler/optimizing/intrinsics_list.h
index 8f1d5e1..96f43a0 100644
--- a/compiler/optimizing/intrinsics_list.h
+++ b/compiler/optimizing/intrinsics_list.h
@@ -51,6 +51,23 @@
V(MathMaxFloatFloat, kStatic, kNeedsEnvironmentOrCache) \
V(MathMaxLongLong, kStatic, kNeedsEnvironmentOrCache) \
V(MathMaxIntInt, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathCos, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathSin, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathAcos, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathAsin, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathAtan, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathAtan2, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathCbrt, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathCosh, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathExp, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathExpm1, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathHypot, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathLog, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathLog10, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathNextAfter, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathSinh, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathTan, kStatic, kNeedsEnvironmentOrCache) \
+ V(MathTanh, kStatic, kNeedsEnvironmentOrCache) \
V(MathSqrt, kStatic, kNeedsEnvironmentOrCache) \
V(MathCeil, kStatic, kNeedsEnvironmentOrCache) \
V(MathFloor, kStatic, kNeedsEnvironmentOrCache) \
diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc
index 9ecce0e..06fab61 100644
--- a/compiler/optimizing/intrinsics_mips.cc
+++ b/compiler/optimizing/intrinsics_mips.cc
@@ -989,6 +989,23 @@
UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
UNIMPLEMENTED_INTRINSIC(SystemArrayCopy)
+UNIMPLEMENTED_INTRINSIC(MathCos)
+UNIMPLEMENTED_INTRINSIC(MathSin)
+UNIMPLEMENTED_INTRINSIC(MathAcos)
+UNIMPLEMENTED_INTRINSIC(MathAsin)
+UNIMPLEMENTED_INTRINSIC(MathAtan)
+UNIMPLEMENTED_INTRINSIC(MathAtan2)
+UNIMPLEMENTED_INTRINSIC(MathCbrt)
+UNIMPLEMENTED_INTRINSIC(MathCosh)
+UNIMPLEMENTED_INTRINSIC(MathExp)
+UNIMPLEMENTED_INTRINSIC(MathExpm1)
+UNIMPLEMENTED_INTRINSIC(MathHypot)
+UNIMPLEMENTED_INTRINSIC(MathLog)
+UNIMPLEMENTED_INTRINSIC(MathLog10)
+UNIMPLEMENTED_INTRINSIC(MathNextAfter)
+UNIMPLEMENTED_INTRINSIC(MathSinh)
+UNIMPLEMENTED_INTRINSIC(MathTan)
+UNIMPLEMENTED_INTRINSIC(MathTanh)
#undef UNIMPLEMENTED_INTRINSIC
#undef __
diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc
index 36e1b20..8aa7d9f 100644
--- a/compiler/optimizing/intrinsics_mips64.cc
+++ b/compiler/optimizing/intrinsics_mips64.cc
@@ -1730,6 +1730,24 @@
UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
UNIMPLEMENTED_INTRINSIC(SystemArrayCopy)
+UNIMPLEMENTED_INTRINSIC(MathCos)
+UNIMPLEMENTED_INTRINSIC(MathSin)
+UNIMPLEMENTED_INTRINSIC(MathAcos)
+UNIMPLEMENTED_INTRINSIC(MathAsin)
+UNIMPLEMENTED_INTRINSIC(MathAtan)
+UNIMPLEMENTED_INTRINSIC(MathAtan2)
+UNIMPLEMENTED_INTRINSIC(MathCbrt)
+UNIMPLEMENTED_INTRINSIC(MathCosh)
+UNIMPLEMENTED_INTRINSIC(MathExp)
+UNIMPLEMENTED_INTRINSIC(MathExpm1)
+UNIMPLEMENTED_INTRINSIC(MathHypot)
+UNIMPLEMENTED_INTRINSIC(MathLog)
+UNIMPLEMENTED_INTRINSIC(MathLog10)
+UNIMPLEMENTED_INTRINSIC(MathNextAfter)
+UNIMPLEMENTED_INTRINSIC(MathSinh)
+UNIMPLEMENTED_INTRINSIC(MathTan)
+UNIMPLEMENTED_INTRINSIC(MathTanh)
+
#undef UNIMPLEMENTED_INTRINSIC
#undef __
diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc
index 5b67cde..3370e73 100644
--- a/compiler/optimizing/intrinsics_x86.cc
+++ b/compiler/optimizing/intrinsics_x86.cc
@@ -788,6 +788,195 @@
__ Bind(&done);
}
+static void CreateFPToFPCallLocations(ArenaAllocator* arena,
+ HInvoke* invoke) {
+ LocationSummary* locations = new (arena) LocationSummary(invoke,
+ LocationSummary::kCall,
+ kIntrinsified);
+ InvokeRuntimeCallingConvention calling_convention;
+ locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0)));
+ locations->SetOut(Location::FpuRegisterLocation(XMM0));
+}
+
+static void GenFPToFPCall(HInvoke* invoke, CodeGeneratorX86* codegen, QuickEntrypointEnum entry) {
+ LocationSummary* locations = invoke->GetLocations();
+ DCHECK(locations->WillCall());
+ DCHECK(invoke->IsInvokeStaticOrDirect());
+ X86Assembler* assembler = codegen->GetAssembler();
+
+ // We need some place to pass the parameters.
+ __ subl(ESP, Immediate(16));
+ __ cfi().AdjustCFAOffset(16);
+
+ // Pass the parameters at the bottom of the stack.
+ __ movsd(Address(ESP, 0), XMM0);
+
+ // If we have a second parameter, pass it next.
+ if (invoke->GetNumberOfArguments() == 2) {
+ __ movsd(Address(ESP, 8), XMM1);
+ }
+
+ // Now do the actual call.
+ __ fs()->call(Address::Absolute(GetThreadOffset<kX86WordSize>(entry)));
+
+ // Extract the return value from the FP stack.
+ __ fstpl(Address(ESP, 0));
+ __ movsd(XMM0, Address(ESP, 0));
+
+ // And clean up the stack.
+ __ addl(ESP, Immediate(16));
+ __ cfi().AdjustCFAOffset(-16);
+
+ codegen->RecordPcInfo(invoke, invoke->GetDexPc());
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathCos(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathCos(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickCos);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathSin(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathSin(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickSin);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathAcos(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathAcos(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAcos);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathAsin(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathAsin(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAsin);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathAtan(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathAtan(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAtan);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathCbrt(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathCbrt(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickCbrt);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathCosh(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathCosh(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickCosh);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathExp(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathExp(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickExp);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathExpm1(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathExpm1(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickExpm1);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathLog(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathLog(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickLog);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathLog10(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathLog10(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickLog10);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathSinh(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathSinh(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickSinh);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathTan(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathTan(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickTan);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathTanh(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathTanh(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickTanh);
+}
+
+static void CreateFPFPToFPCallLocations(ArenaAllocator* arena,
+ HInvoke* invoke) {
+ LocationSummary* locations = new (arena) LocationSummary(invoke,
+ LocationSummary::kCall,
+ kIntrinsified);
+ InvokeRuntimeCallingConvention calling_convention;
+ locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0)));
+ locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1)));
+ locations->SetOut(Location::FpuRegisterLocation(XMM0));
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathAtan2(HInvoke* invoke) {
+ CreateFPFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathAtan2(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAtan2);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathHypot(HInvoke* invoke) {
+ CreateFPFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathHypot(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickHypot);
+}
+
+void IntrinsicLocationsBuilderX86::VisitMathNextAfter(HInvoke* invoke) {
+ CreateFPFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitMathNextAfter(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickNextAfter);
+}
+
void IntrinsicLocationsBuilderX86::VisitStringCharAt(HInvoke* invoke) {
// The inputs plus one temp.
LocationSummary* locations = new (arena_) LocationSummary(invoke,
diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc
index ecd129f..e862573 100644
--- a/compiler/optimizing/intrinsics_x86_64.cc
+++ b/compiler/optimizing/intrinsics_x86_64.cc
@@ -703,6 +703,188 @@
__ Bind(&done);
}
+static void CreateFPToFPCallLocations(ArenaAllocator* arena,
+ HInvoke* invoke) {
+ LocationSummary* locations = new (arena) LocationSummary(invoke,
+ LocationSummary::kCall,
+ kIntrinsified);
+ InvokeRuntimeCallingConvention calling_convention;
+ locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0)));
+ locations->SetOut(Location::FpuRegisterLocation(XMM0));
+
+ // We have to ensure that the native code doesn't clobber the XMM registers which are
+ // non-volatile for ART, but volatile for Native calls. This will ensure that they are
+ // saved in the prologue and properly restored.
+ for (auto fp_reg : non_volatile_xmm_regs) {
+ locations->AddTemp(Location::FpuRegisterLocation(fp_reg));
+ }
+}
+
+static void GenFPToFPCall(HInvoke* invoke, CodeGeneratorX86_64* codegen,
+ QuickEntrypointEnum entry) {
+ LocationSummary* locations = invoke->GetLocations();
+ DCHECK(locations->WillCall());
+ DCHECK(invoke->IsInvokeStaticOrDirect());
+ X86_64Assembler* assembler = codegen->GetAssembler();
+
+ __ gs()->call(Address::Absolute(GetThreadOffset<kX86_64WordSize>(entry), true));
+ codegen->RecordPcInfo(invoke, invoke->GetDexPc());
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathCos(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathCos(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickCos);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathSin(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathSin(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickSin);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathAcos(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathAcos(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAcos);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathAsin(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathAsin(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAsin);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathAtan(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathAtan(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAtan);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathCbrt(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathCbrt(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickCbrt);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathCosh(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathCosh(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickCosh);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathExp(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathExp(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickExp);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathExpm1(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathExpm1(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickExpm1);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathLog(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathLog(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickLog);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathLog10(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathLog10(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickLog10);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathSinh(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathSinh(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickSinh);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathTan(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathTan(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickTan);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathTanh(HInvoke* invoke) {
+ CreateFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathTanh(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickTanh);
+}
+
+static void CreateFPFPToFPCallLocations(ArenaAllocator* arena,
+ HInvoke* invoke) {
+ LocationSummary* locations = new (arena) LocationSummary(invoke,
+ LocationSummary::kCall,
+ kIntrinsified);
+ InvokeRuntimeCallingConvention calling_convention;
+ locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0)));
+ locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1)));
+ locations->SetOut(Location::FpuRegisterLocation(XMM0));
+
+ // We have to ensure that the native code doesn't clobber the XMM registers which are
+ // non-volatile for ART, but volatile for Native calls. This will ensure that they are
+ // saved in the prologue and properly restored.
+ for (auto fp_reg : non_volatile_xmm_regs) {
+ locations->AddTemp(Location::FpuRegisterLocation(fp_reg));
+ }
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathAtan2(HInvoke* invoke) {
+ CreateFPFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathAtan2(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickAtan2);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathHypot(HInvoke* invoke) {
+ CreateFPFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathHypot(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickHypot);
+}
+
+void IntrinsicLocationsBuilderX86_64::VisitMathNextAfter(HInvoke* invoke) {
+ CreateFPFPToFPCallLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitMathNextAfter(HInvoke* invoke) {
+ GenFPToFPCall(invoke, codegen_, kQuickNextAfter);
+}
+
void IntrinsicLocationsBuilderX86_64::VisitStringCharAt(HInvoke* invoke) {
// The inputs plus one temp.
LocationSummary* locations = new (arena_) LocationSummary(invoke,