ART: Implement HSelect with CSEL/FCSEL on arm64
Change-Id: I549af0cba3c5048066a2d1206b78a70b496d349e
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index de23fe8..435ae5e 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -2928,30 +2928,128 @@
/* false_target */ nullptr);
}
+enum SelectVariant {
+ kCsel,
+ kCselFalseConst,
+ kCselTrueConst,
+ kFcsel,
+};
+
+static inline bool IsConditionOnFloatingPointValues(HInstruction* condition) {
+ return condition->IsCondition() &&
+ Primitive::IsFloatingPointType(condition->InputAt(0)->GetType());
+}
+
+static inline bool IsRecognizedCselConstant(HInstruction* constant) {
+ if (constant->IsConstant()) {
+ int64_t value = Int64FromConstant(constant->AsConstant());
+ if ((value == -1) || (value == 0) || (value == 1)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static inline SelectVariant GetSelectVariant(HSelect* select) {
+ if (Primitive::IsFloatingPointType(select->GetType())) {
+ return kFcsel;
+ } else if (IsRecognizedCselConstant(select->GetFalseValue())) {
+ return kCselFalseConst;
+ } else if (IsRecognizedCselConstant(select->GetTrueValue())) {
+ return kCselTrueConst;
+ } else {
+ return kCsel;
+ }
+}
+
+static inline bool HasSwappedInputs(SelectVariant variant) {
+ return variant == kCselTrueConst;
+}
+
+static inline Condition GetConditionForSelect(HCondition* condition, SelectVariant variant) {
+ IfCondition cond = HasSwappedInputs(variant) ? condition->GetOppositeCondition()
+ : condition->GetCondition();
+ return IsConditionOnFloatingPointValues(condition) ? ARM64FPCondition(cond, condition->IsGtBias())
+ : ARM64Condition(cond);
+}
+
void LocationsBuilderARM64::VisitSelect(HSelect* select) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
- if (Primitive::IsFloatingPointType(select->GetType())) {
- locations->SetInAt(0, Location::RequiresFpuRegister());
- locations->SetInAt(1, Location::RequiresFpuRegister());
- } else {
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
+ switch (GetSelectVariant(select)) {
+ case kCsel:
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetOut(Location::RequiresRegister());
+ break;
+ case kCselFalseConst:
+ locations->SetInAt(0, Location::ConstantLocation(select->InputAt(0)->AsConstant()));
+ locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetOut(Location::RequiresRegister());
+ break;
+ case kCselTrueConst:
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::ConstantLocation(select->InputAt(1)->AsConstant()));
+ locations->SetOut(Location::RequiresRegister());
+ break;
+ case kFcsel:
+ locations->SetInAt(0, Location::RequiresFpuRegister());
+ locations->SetInAt(1, Location::RequiresFpuRegister());
+ locations->SetOut(Location::RequiresFpuRegister());
+ break;
}
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
locations->SetInAt(2, Location::RequiresRegister());
}
- locations->SetOut(Location::SameAsFirstInput());
}
void InstructionCodeGeneratorARM64::VisitSelect(HSelect* select) {
- LocationSummary* locations = select->GetLocations();
- vixl::Label false_target;
- GenerateTestAndBranch(select,
- /* condition_input_index */ 2,
- /* true_target */ nullptr,
- &false_target);
- codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
- __ Bind(&false_target);
+ HInstruction* cond = select->GetCondition();
+ SelectVariant variant = GetSelectVariant(select);
+ Condition csel_cond;
+
+ if (IsBooleanValueOrMaterializedCondition(cond)) {
+ if (cond->IsCondition() && cond->GetNext() == select) {
+ // Condition codes set from previous instruction.
+ csel_cond = GetConditionForSelect(cond->AsCondition(), variant);
+ } else {
+ __ Cmp(InputRegisterAt(select, 2), 0);
+ csel_cond = HasSwappedInputs(variant) ? eq : ne;
+ }
+ } else if (IsConditionOnFloatingPointValues(cond)) {
+ Location rhs = cond->GetLocations()->InAt(1);
+ if (rhs.IsConstant()) {
+ DCHECK(IsFloatingPointZeroConstant(rhs.GetConstant()));
+ __ Fcmp(InputFPRegisterAt(cond, 0), 0.0);
+ } else {
+ __ Fcmp(InputFPRegisterAt(cond, 0), InputFPRegisterAt(cond, 1));
+ }
+ csel_cond = GetConditionForSelect(cond->AsCondition(), variant);
+ } else {
+ __ Cmp(InputRegisterAt(cond, 0), InputOperandAt(cond, 1));
+ csel_cond = GetConditionForSelect(cond->AsCondition(), variant);
+ }
+
+ switch (variant) {
+ case kCsel:
+ case kCselFalseConst:
+ __ Csel(OutputRegister(select),
+ InputRegisterAt(select, 1),
+ InputOperandAt(select, 0),
+ csel_cond);
+ break;
+ case kCselTrueConst:
+ __ Csel(OutputRegister(select),
+ InputRegisterAt(select, 0),
+ InputOperandAt(select, 1),
+ csel_cond);
+ break;
+ case kFcsel:
+ __ Fcsel(OutputFPRegister(select),
+ InputFPRegisterAt(select, 1),
+ InputFPRegisterAt(select, 0),
+ csel_cond);
+ break;
+ }
}
void LocationsBuilderARM64::VisitNativeDebugInfo(HNativeDebugInfo* info) {
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index 2697af3..323cfae 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -5997,9 +5997,14 @@
};
inline int64_t Int64FromConstant(HConstant* constant) {
- DCHECK(constant->IsIntConstant() || constant->IsLongConstant());
- return constant->IsIntConstant() ? constant->AsIntConstant()->GetValue()
- : constant->AsLongConstant()->GetValue();
+ if (constant->IsIntConstant()) {
+ return constant->AsIntConstant()->GetValue();
+ } else if (constant->IsLongConstant()) {
+ return constant->AsLongConstant()->GetValue();
+ } else {
+ DCHECK(constant->IsNullConstant());
+ return 0;
+ }
}
inline bool IsSameDexFile(const DexFile& lhs, const DexFile& rhs) {
diff --git a/test/570-checker-select/src/Main.java b/test/570-checker-select/src/Main.java
index ec60240..8a4cf60 100644
--- a/test/570-checker-select/src/Main.java
+++ b/test/570-checker-select/src/Main.java
@@ -19,6 +19,11 @@
/// CHECK-START: int Main.BoolCond_IntVarVar(boolean, int, int) register (after)
/// CHECK: Select [{{i\d+}},{{i\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: int Main.BoolCond_IntVarVar(boolean, int, int) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csel ne
+
/// CHECK-START-X86_64: int Main.BoolCond_IntVarVar(boolean, int, int) disassembly (after)
/// CHECK: <<Cond:z\d+>> ParameterValue
/// CHECK: Select [{{i\d+}},{{i\d+}},<<Cond>>]
@@ -31,6 +36,11 @@
/// CHECK-START: int Main.BoolCond_IntVarCst(boolean, int) register (after)
/// CHECK: Select [{{i\d+}},{{i\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: int Main.BoolCond_IntVarCst(boolean, int) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csinc ne
+
/// CHECK-START-X86_64: int Main.BoolCond_IntVarCst(boolean, int) disassembly (after)
/// CHECK: <<Cond:z\d+>> ParameterValue
/// CHECK: Select [{{i\d+}},{{i\d+}},<<Cond>>]
@@ -43,6 +53,11 @@
/// CHECK-START: int Main.BoolCond_IntCstVar(boolean, int) register (after)
/// CHECK: Select [{{i\d+}},{{i\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: int Main.BoolCond_IntCstVar(boolean, int) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csinc eq
+
/// CHECK-START-X86_64: int Main.BoolCond_IntCstVar(boolean, int) disassembly (after)
/// CHECK: <<Cond:z\d+>> ParameterValue
/// CHECK: Select [{{i\d+}},{{i\d+}},<<Cond>>]
@@ -55,6 +70,11 @@
/// CHECK-START: long Main.BoolCond_LongVarVar(boolean, long, long) register (after)
/// CHECK: Select [{{j\d+}},{{j\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: long Main.BoolCond_LongVarVar(boolean, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csel ne
+
/// CHECK-START-X86_64: long Main.BoolCond_LongVarVar(boolean, long, long) disassembly (after)
/// CHECK: <<Cond:z\d+>> ParameterValue
/// CHECK: Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -67,6 +87,11 @@
/// CHECK-START: long Main.BoolCond_LongVarCst(boolean, long) register (after)
/// CHECK: Select [{{j\d+}},{{j\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: long Main.BoolCond_LongVarCst(boolean, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csinc ne
+
/// CHECK-START-X86_64: long Main.BoolCond_LongVarCst(boolean, long) disassembly (after)
/// CHECK: <<Cond:z\d+>> ParameterValue
/// CHECK: Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -79,6 +104,11 @@
/// CHECK-START: long Main.BoolCond_LongCstVar(boolean, long) register (after)
/// CHECK: Select [{{j\d+}},{{j\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: long Main.BoolCond_LongCstVar(boolean, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csinc eq
+
/// CHECK-START-X86_64: long Main.BoolCond_LongCstVar(boolean, long) disassembly (after)
/// CHECK: <<Cond:z\d+>> ParameterValue
/// CHECK: Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -91,6 +121,11 @@
/// CHECK-START: float Main.BoolCond_FloatVarVar(boolean, float, float) register (after)
/// CHECK: Select [{{f\d+}},{{f\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: float Main.BoolCond_FloatVarVar(boolean, float, float) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: fcsel ne
+
public static float BoolCond_FloatVarVar(boolean cond, float x, float y) {
return cond ? x : y;
}
@@ -98,6 +133,11 @@
/// CHECK-START: float Main.BoolCond_FloatVarCst(boolean, float) register (after)
/// CHECK: Select [{{f\d+}},{{f\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: float Main.BoolCond_FloatVarCst(boolean, float) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: fcsel ne
+
public static float BoolCond_FloatVarCst(boolean cond, float x) {
return cond ? x : 1.0f;
}
@@ -105,6 +145,11 @@
/// CHECK-START: float Main.BoolCond_FloatCstVar(boolean, float) register (after)
/// CHECK: Select [{{f\d+}},{{f\d+}},{{z\d+}}]
+ /// CHECK-START-ARM64: float Main.BoolCond_FloatCstVar(boolean, float) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: fcsel ne
+
public static float BoolCond_FloatCstVar(boolean cond, float y) {
return cond ? 1.0f : y;
}
@@ -113,6 +158,11 @@
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{i\d+}},{{i\d+}}]
/// CHECK-NEXT: Select [{{i\d+}},{{i\d+}},<<Cond>>]
+ /// CHECK-START-ARM64: int Main.IntNonmatCond_IntVarVar(int, int, int, int) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csel le
+
/// CHECK-START-X86_64: int Main.IntNonmatCond_IntVarVar(int, int, int, int) disassembly (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{i\d+}},{{i\d+}}]
/// CHECK-NEXT: Select [{{i\d+}},{{i\d+}},<<Cond>>]
@@ -127,6 +177,13 @@
/// CHECK-NEXT: <<Sel:i\d+>> Select [{{i\d+}},{{i\d+}},{{z\d+}}]
/// CHECK-NEXT: Add [<<Cond>>,<<Sel>>]
+ /// CHECK-START-ARM64: int Main.IntMatCond_IntVarVar(int, int, int, int) disassembly (after)
+ /// CHECK: LessThanOrEqual
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: cset le
+ /// CHECK: Select
+ /// CHECK-NEXT: csel le
+
/// CHECK-START-X86_64: int Main.IntMatCond_IntVarVar(int, int, int, int) disassembly (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{i\d+}},{{i\d+}}]
/// CHECK: Select [{{i\d+}},{{i\d+}},<<Cond>>]
@@ -141,6 +198,11 @@
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{i\d+}},{{i\d+}}]
/// CHECK-NEXT: Select [{{j\d+}},{{j\d+}},<<Cond>>]
+ /// CHECK-START-ARM64: long Main.IntNonmatCond_LongVarVar(int, int, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csel le
+
/// CHECK-START-X86_64: long Main.IntNonmatCond_LongVarVar(int, int, long, long) disassembly (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{i\d+}},{{i\d+}}]
/// CHECK-NEXT: Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -156,6 +218,13 @@
/// CHECK: <<Sel2:j\d+>> Select [{{j\d+}},{{j\d+}},<<Cond>>]
/// CHECK: Add [<<Sel2>>,<<Sel1>>]
+ /// CHECK-START-ARM64: long Main.IntMatCond_LongVarVar(int, int, long, long) disassembly (after)
+ /// CHECK: LessThanOrEqual
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: cset le
+ /// CHECK: Select
+ /// CHECK-NEXT: csel le
+
/// CHECK-START-X86_64: long Main.IntMatCond_LongVarVar(int, int, long, long) disassembly (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{i\d+}},{{i\d+}}]
/// CHECK: Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -172,6 +241,11 @@
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{j\d+}},{{j\d+}}]
/// CHECK: Select [{{j\d+}},{{j\d+}},<<Cond>>]
+ /// CHECK-START-ARM64: long Main.LongNonmatCond_LongVarVar(long, long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: csel le
+
/// CHECK-START-X86_64: long Main.LongNonmatCond_LongVarVar(long, long, long, long) disassembly (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{j\d+}},{{j\d+}}]
/// CHECK: Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -187,6 +261,13 @@
/// CHECK: <<Sel2:j\d+>> Select [{{j\d+}},{{j\d+}},<<Cond>>]
/// CHECK: Add [<<Sel2>>,<<Sel1>>]
+ /// CHECK-START-ARM64: long Main.LongMatCond_LongVarVar(long, long, long, long) disassembly (after)
+ /// CHECK: LessThanOrEqual
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: cset le
+ /// CHECK: Select
+ /// CHECK-NEXT: csel le
+
/// CHECK-START-X86_64: long Main.LongMatCond_LongVarVar(long, long, long, long) disassembly (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{j\d+}},{{j\d+}}]
/// CHECK: Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -203,6 +284,12 @@
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{f\d+}},{{f\d+}}]
/// CHECK-NEXT: Select [{{i\d+}},{{i\d+}},<<Cond>>]
+ /// CHECK-START-ARM64: int Main.FloatLtNonmatCond_IntVarVar(float, float, int, int) disassembly (after)
+ /// CHECK: LessThanOrEqual
+ /// CHECK: Select
+ /// CHECK-NEXT: fcmp
+ /// CHECK-NEXT: csel le
+
public static int FloatLtNonmatCond_IntVarVar(float a, float b, int x, int y) {
return a > b ? x : y;
}
@@ -211,6 +298,12 @@
/// CHECK: <<Cond:z\d+>> GreaterThanOrEqual [{{f\d+}},{{f\d+}}]
/// CHECK-NEXT: Select [{{i\d+}},{{i\d+}},<<Cond>>]
+ /// CHECK-START-ARM64: int Main.FloatGtNonmatCond_IntVarVar(float, float, int, int) disassembly (after)
+ /// CHECK: GreaterThanOrEqual
+ /// CHECK: Select
+ /// CHECK-NEXT: fcmp
+ /// CHECK-NEXT: csel hs
+
public static int FloatGtNonmatCond_IntVarVar(float a, float b, int x, int y) {
return a < b ? x : y;
}
@@ -219,6 +312,12 @@
/// CHECK: <<Cond:z\d+>> GreaterThanOrEqual [{{f\d+}},{{f\d+}}]
/// CHECK-NEXT: Select [{{f\d+}},{{f\d+}},<<Cond>>]
+ /// CHECK-START-ARM64: float Main.FloatGtNonmatCond_FloatVarVar(float, float, float, float) disassembly (after)
+ /// CHECK: GreaterThanOrEqual
+ /// CHECK: Select
+ /// CHECK-NEXT: fcmp
+ /// CHECK-NEXT: fcsel hs
+
public static float FloatGtNonmatCond_FloatVarVar(float a, float b, float x, float y) {
return a < b ? x : y;
}
@@ -228,6 +327,13 @@
/// CHECK-NEXT: <<Sel:i\d+>> Select [{{i\d+}},{{i\d+}},<<Cond>>]
/// CHECK-NEXT: Add [<<Cond>>,<<Sel>>]
+ /// CHECK-START-ARM64: int Main.FloatLtMatCond_IntVarVar(float, float, int, int) disassembly (after)
+ /// CHECK: LessThanOrEqual
+ /// CHECK-NEXT: fcmp
+ /// CHECK-NEXT: cset le
+ /// CHECK: Select
+ /// CHECK-NEXT: csel le
+
public static int FloatLtMatCond_IntVarVar(float a, float b, int x, int y) {
int result = (a > b ? x : y);
return result + (a > b ? 0 : 1);
@@ -238,6 +344,13 @@
/// CHECK-NEXT: <<Sel:i\d+>> Select [{{i\d+}},{{i\d+}},<<Cond>>]
/// CHECK-NEXT: Add [<<Cond>>,<<Sel>>]
+ /// CHECK-START-ARM64: int Main.FloatGtMatCond_IntVarVar(float, float, int, int) disassembly (after)
+ /// CHECK: GreaterThanOrEqual
+ /// CHECK-NEXT: fcmp
+ /// CHECK-NEXT: cset hs
+ /// CHECK: Select
+ /// CHECK-NEXT: csel hs
+
public static int FloatGtMatCond_IntVarVar(float a, float b, int x, int y) {
int result = (a < b ? x : y);
return result + (a < b ? 0 : 1);
@@ -248,6 +361,13 @@
/// CHECK-NEXT: <<Sel:f\d+>> Select [{{f\d+}},{{f\d+}},<<Cond>>]
/// CHECK-NEXT: TypeConversion [<<Cond>>]
+ /// CHECK-START-ARM64: float Main.FloatGtMatCond_FloatVarVar(float, float, float, float) disassembly (after)
+ /// CHECK: GreaterThanOrEqual
+ /// CHECK-NEXT: fcmp
+ /// CHECK-NEXT: cset hs
+ /// CHECK: Select
+ /// CHECK-NEXT: fcsel hs
+
public static float FloatGtMatCond_FloatVarVar(float a, float b, float x, float y) {
float result = (a < b ? x : y);
return result + (a < b ? 0 : 1);