ART: Introduce predicated vector instructions.
This CL introduces a minimal changes to the IR to support
autovectorization with use of predicated execution of SIMD
instructions (e.g. Arm SVE).
Test: test-art-target, test-art-host.
Change-Id: Ibb7c5520fec6b858fb29f0dde19ec65501831a3a
diff --git a/compiler/optimizing/code_generator_vector_arm64_neon.cc b/compiler/optimizing/code_generator_vector_arm64_neon.cc
index 78720c3..714d984 100644
--- a/compiler/optimizing/code_generator_vector_arm64_neon.cc
+++ b/compiler/optimizing/code_generator_vector_arm64_neon.cc
@@ -1486,6 +1486,36 @@
}
}
+void LocationsBuilderARM64Neon::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARM64Neon::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderARM64Neon::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARM64Neon::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderARM64Neon::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARM64Neon::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
Location InstructionCodeGeneratorARM64Neon::AllocateSIMDScratchLocation(
vixl::aarch64::UseScratchRegisterScope* scope) {
DCHECK_EQ(codegen_->GetSIMDRegisterWidth(), kQRegSizeInBytes);
diff --git a/compiler/optimizing/code_generator_vector_arm64_sve.cc b/compiler/optimizing/code_generator_vector_arm64_sve.cc
index 5460ff2..d6fa0f6 100644
--- a/compiler/optimizing/code_generator_vector_arm64_sve.cc
+++ b/compiler/optimizing/code_generator_vector_arm64_sve.cc
@@ -1486,6 +1486,36 @@
}
}
+void LocationsBuilderARM64Sve::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARM64Sve::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderARM64Sve::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARM64Sve::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderARM64Sve::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARM64Sve::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
Location InstructionCodeGeneratorARM64Sve::AllocateSIMDScratchLocation(
vixl::aarch64::UseScratchRegisterScope* scope) {
DCHECK_EQ(codegen_->GetSIMDRegisterWidth(), kQRegSizeInBytes);
diff --git a/compiler/optimizing/code_generator_vector_arm_vixl.cc b/compiler/optimizing/code_generator_vector_arm_vixl.cc
index b092961..33f2491 100644
--- a/compiler/optimizing/code_generator_vector_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_vector_arm_vixl.cc
@@ -1048,6 +1048,36 @@
}
}
+void LocationsBuilderARMVIXL::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARMVIXL::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderARMVIXL::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARMVIXL::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderARMVIXL::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorARMVIXL::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
#undef __
} // namespace arm
diff --git a/compiler/optimizing/code_generator_vector_x86.cc b/compiler/optimizing/code_generator_vector_x86.cc
index 1390af2..9c837dd 100644
--- a/compiler/optimizing/code_generator_vector_x86.cc
+++ b/compiler/optimizing/code_generator_vector_x86.cc
@@ -1381,6 +1381,36 @@
}
}
+void LocationsBuilderX86::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorX86::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderX86::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorX86::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderX86::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorX86::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
#undef __
} // namespace x86
diff --git a/compiler/optimizing/code_generator_vector_x86_64.cc b/compiler/optimizing/code_generator_vector_x86_64.cc
index 7fac44d..330bf76 100644
--- a/compiler/optimizing/code_generator_vector_x86_64.cc
+++ b/compiler/optimizing/code_generator_vector_x86_64.cc
@@ -1354,6 +1354,36 @@
}
}
+void LocationsBuilderX86_64::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorX86_64::VisitVecPredSetAll(HVecPredSetAll* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderX86_64::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorX86_64::VisitVecPredWhile(HVecPredWhile* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void LocationsBuilderX86_64::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
+void InstructionCodeGeneratorX86_64::VisitVecPredCondition(HVecPredCondition* instruction) {
+ LOG(FATAL) << "No SIMD for " << instruction->GetId();
+ UNREACHABLE();
+}
+
#undef __
} // namespace x86_64
diff --git a/compiler/optimizing/graph_checker.cc b/compiler/optimizing/graph_checker.cc
index 95cfe3e..ece88a0 100644
--- a/compiler/optimizing/graph_checker.cc
+++ b/compiler/optimizing/graph_checker.cc
@@ -25,6 +25,7 @@
#include "base/bit_vector-inl.h"
#include "base/scoped_arena_allocator.h"
#include "base/scoped_arena_containers.h"
+#include "code_generator.h"
#include "handle.h"
#include "mirror/class.h"
#include "obj_ptr-inl.h"
@@ -1141,4 +1142,26 @@
}
}
+void GraphChecker::VisitVecOperation(HVecOperation* instruction) {
+ VisitInstruction(instruction);
+ if (codegen_ == nullptr) {
+ return;
+ }
+
+ if (!codegen_->SupportsPredicatedSIMD() && instruction->IsPredicated()) {
+ AddError(StringPrintf(
+ "%s %d must not be predicated.",
+ instruction->DebugName(),
+ instruction->GetId()));
+ }
+
+ if (codegen_->SupportsPredicatedSIMD() &&
+ (instruction->MustBePredicatedInPredicatedSIMDMode() != instruction->IsPredicated())) {
+ AddError(StringPrintf(
+ "%s %d predication mode is incorrect; see HVecOperation::MustBePredicated.",
+ instruction->DebugName(),
+ instruction->GetId()));
+ }
+}
+
} // namespace art
diff --git a/compiler/optimizing/graph_checker.h b/compiler/optimizing/graph_checker.h
index 564b137..04c8d21 100644
--- a/compiler/optimizing/graph_checker.h
+++ b/compiler/optimizing/graph_checker.h
@@ -26,15 +26,20 @@
namespace art {
+class CodeGenerator;
+
// A control-flow graph visitor performing various checks.
class GraphChecker : public HGraphDelegateVisitor {
public:
- explicit GraphChecker(HGraph* graph, const char* dump_prefix = "art::GraphChecker: ")
+ explicit GraphChecker(HGraph* graph,
+ CodeGenerator* codegen = nullptr,
+ const char* dump_prefix = "art::GraphChecker: ")
: HGraphDelegateVisitor(graph),
errors_(graph->GetAllocator()->Adapter(kArenaAllocGraphChecker)),
dump_prefix_(dump_prefix),
allocator_(graph->GetArenaStack()),
- seen_ids_(&allocator_, graph->GetCurrentInstructionId(), false, kArenaAllocGraphChecker) {
+ seen_ids_(&allocator_, graph->GetCurrentInstructionId(), false, kArenaAllocGraphChecker),
+ codegen_(codegen) {
seen_ids_.ClearAllBits();
}
@@ -69,6 +74,8 @@
void VisitTryBoundary(HTryBoundary* try_boundary) override;
void VisitTypeConversion(HTypeConversion* instruction) override;
+ void VisitVecOperation(HVecOperation* instruction) override;
+
void CheckTypeCheckBitstringInput(HTypeCheckInstruction* check,
size_t input_pos,
bool check_value,
@@ -125,6 +132,9 @@
// The default value is true.
bool check_reference_type_info_ = true;
+ // Used to access target information.
+ CodeGenerator* codegen_;
+
DISALLOW_COPY_AND_ASSIGN(GraphChecker);
};
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index 7c0e973..00dc50c 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -1557,6 +1557,9 @@
M(VecDotProd, VecOperation) \
M(VecLoad, VecMemoryOperation) \
M(VecStore, VecMemoryOperation) \
+ M(VecPredSetAll, VecPredSetOperation) \
+ M(VecPredWhile, VecPredSetOperation) \
+ M(VecPredCondition, VecOperation) \
#define FOR_EACH_CONCRETE_INSTRUCTION_COMMON(M) \
FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(M) \
@@ -1617,7 +1620,8 @@
M(VecOperation, Instruction) \
M(VecUnaryOperation, VecOperation) \
M(VecBinaryOperation, VecOperation) \
- M(VecMemoryOperation, VecOperation)
+ M(VecMemoryOperation, VecOperation) \
+ M(VecPredSetOperation, VecOperation)
#define FOR_EACH_INSTRUCTION(M) \
FOR_EACH_CONCRETE_INSTRUCTION(M) \
diff --git a/compiler/optimizing/nodes_vector.h b/compiler/optimizing/nodes_vector.h
index e817048..9c6b422 100644
--- a/compiler/optimizing/nodes_vector.h
+++ b/compiler/optimizing/nodes_vector.h
@@ -87,9 +87,64 @@
kArenaAllocVectorNode),
vector_length_(vector_length) {
SetPackedField<PackedTypeField>(packed_type);
+ // By default vector operations are not predicated.
+ SetPackedField<PredicationKindField>(PredicationKind::kNotPredicated);
DCHECK_LT(1u, vector_length);
}
+ // Predicated instructions execute a corresponding operation only on vector elements which are
+ // active (governing predicate is true for that element); the following modes determine what
+ // is happening with inactive elements.
+ //
+ // See HVecPredSetOperation.
+ enum class PredicationKind {
+ kNotPredicated, // Instruction doesn't take any predicate as an input.
+ kZeroingForm, // Inactive elements are reset to zero.
+ kMergingForm, // Inactive elements keep their value.
+ kLast = kMergingForm,
+ };
+
+ PredicationKind GetPredicationKind() const { return GetPackedField<PredicationKindField>(); }
+
+ // Returns whether the vector operation must be predicated in predicated SIMD mode
+ // (see CodeGenerator::SupportsPredicatedSIMD). The method reflects semantics of
+ // the instruction class rather than the state of a particular instruction instance.
+ //
+ // This property is introduced for robustness purpose - to maintain and check the invariant:
+ // all instructions of the same vector operation class must be either all predicated or all
+ // not predicated (depending on the predicated SIMD support) in a correct graph.
+ virtual bool MustBePredicatedInPredicatedSIMDMode() {
+ return true;
+ }
+
+ bool IsPredicated() const {
+ return GetPredicationKind() != PredicationKind::kNotPredicated;
+ }
+
+ // See HVecPredSetOperation.
+ void SetGoverningPredicate(HInstruction* input, PredicationKind pred_kind) {
+ DCHECK(!IsPredicated());
+ DCHECK(input->IsVecPredSetOperation());
+ AddInput(input);
+ SetPackedField<PredicationKindField>(pred_kind);
+ DCHECK(IsPredicated());
+ }
+
+ void SetMergingGoverningPredicate(HInstruction* input) {
+ SetGoverningPredicate(input, PredicationKind::kMergingForm);
+ }
+ void SetZeroingGoverningPredicate(HInstruction* input) {
+ SetGoverningPredicate(input, PredicationKind::kZeroingForm);
+ }
+
+ // See HVecPredSetOperation.
+ HVecPredSetOperation* GetGoverningPredicate() const {
+ DCHECK(IsPredicated());
+ HInstruction* pred_input = InputAt(InputCount() - 1);
+ DCHECK(pred_input->IsVecPredSetOperation());
+ return pred_input->AsVecPredSetOperation();
+ }
+
// Returns the number of elements packed in a vector.
size_t GetVectorLength() const {
return vector_length_;
@@ -181,12 +236,16 @@
protected:
// Additional packed bits.
- static constexpr size_t kFieldPackedType = HInstruction::kNumberOfGenericPackedBits;
+ static constexpr size_t kPredicationKind = HInstruction::kNumberOfGenericPackedBits;
+ static constexpr size_t kPredicationKindSize =
+ MinimumBitsToStore(static_cast<size_t>(PredicationKind::kLast));
+ static constexpr size_t kFieldPackedType = kPredicationKind + kPredicationKindSize;
static constexpr size_t kFieldPackedTypeSize =
MinimumBitsToStore(static_cast<size_t>(DataType::Type::kLast));
static constexpr size_t kNumberOfVectorOpPackedBits = kFieldPackedType + kFieldPackedTypeSize;
static_assert(kNumberOfVectorOpPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields.");
using PackedTypeField = BitField<DataType::Type, kFieldPackedType, kFieldPackedTypeSize>;
+ using PredicationKindField = BitField<PredicationKind, kPredicationKind, kPredicationKindSize>;
DEFAULT_COPY_CONSTRUCTOR(VecOperation);
@@ -1163,6 +1222,237 @@
DEFAULT_COPY_CONSTRUCTOR(VecStore)
};
+//
+// 'Predicate-setting' instructions.
+//
+
+// An abstract class for instructions for which the output value is a vector predicate -
+// a special kind of vector value:
+//
+// viz. [ p1, .. , pn ], where p_i is from { 0, 1 }.
+//
+// A VecOperation OP executes the same operation (e.g. ADD) on multiple elements of the vector.
+// It can be either unpredicated (operation is done on ALL of the elements) or predicated (only
+// on SOME elements, determined by a special extra input - vector predicate).
+// Implementations can vary depending on the ISA; the general idea is that for each element of the
+// regular vector a vector predicate has a corresponding element with either 0 or 1.
+// The value determines whether a vector element will be involved in OP calculations or not
+// (active or inactive). A vector predicate is referred as governing one if it is used to
+// control the execution of a predicated instruction.
+//
+// Note: vector predicate value type is introduced alongside existing vectors of booleans and
+// vectors of bytes to reflect their special semantics.
+//
+// TODO: we could introduce SIMD types in HIR.
+class HVecPredSetOperation : public HVecOperation {
+ public:
+ // A vector predicate-setting operation looks like a Int64 location.
+ // TODO: we could introduce vector types in HIR.
+ static constexpr DataType::Type kSIMDPredType = DataType::Type::kInt64;
+
+ HVecPredSetOperation(InstructionKind kind,
+ ArenaAllocator* allocator,
+ DataType::Type packed_type,
+ SideEffects side_effects,
+ size_t number_of_inputs,
+ size_t vector_length,
+ uint32_t dex_pc)
+ : HVecOperation(kind,
+ allocator,
+ packed_type,
+ side_effects,
+ number_of_inputs,
+ vector_length,
+ dex_pc) {
+ // Overrides the kSIMDType set by the VecOperation constructor.
+ SetPackedField<TypeField>(kSIMDPredType);
+ }
+
+ bool CanBeMoved() const override { return true; }
+
+ DECLARE_ABSTRACT_INSTRUCTION(VecPredSetOperation);
+
+ protected:
+ DEFAULT_COPY_CONSTRUCTOR(VecPredSetOperation);
+};
+
+// Sets all the vector predicate elements as active or inactive.
+//
+// viz. [ p1, .. , pn ] = [ val, .. , val ] where val is from { 1, 0 }.
+class HVecPredSetAll final : public HVecPredSetOperation {
+ public:
+ HVecPredSetAll(ArenaAllocator* allocator,
+ HInstruction* input,
+ DataType::Type packed_type,
+ size_t vector_length,
+ uint32_t dex_pc) :
+ HVecPredSetOperation(kVecPredSetAll,
+ allocator,
+ packed_type,
+ SideEffects::None(),
+ /* number_of_inputs= */ 1,
+ vector_length,
+ dex_pc) {
+ DCHECK(input->IsIntConstant());
+ SetRawInputAt(0, input);
+ MarkEmittedAtUseSite();
+ }
+
+ // Having governing predicate doesn't make sense for set all TRUE/FALSE instruction.
+ bool MustBePredicatedInPredicatedSIMDMode() override { return false; }
+
+ bool IsSetTrue() const { return InputAt(0)->AsIntConstant()->IsTrue(); }
+
+ // Vector predicates are not kept alive across vector loop boundaries.
+ bool CanBeMoved() const override { return false; }
+
+ DECLARE_INSTRUCTION(VecPredSetAll);
+
+ protected:
+ DEFAULT_COPY_CONSTRUCTOR(VecPredSetAll);
+};
+
+//
+// Arm64 SVE-specific instructions.
+//
+// Classes of instructions which are specific to Arm64 SVE (though could be adopted
+// by other targets, possibly being lowered to a number of ISA instructions) and
+// implement SIMD loop predicated execution idiom.
+//
+
+// Takes two scalar values x and y, creates a vector S: s(n) = x + n, compares (OP) each s(n)
+// with y and set the corresponding element of the predicate register to the result of the
+// comparison.
+//
+// viz. [ p1, .. , pn ] = [ x OP y , (x + 1) OP y, .. , (x + n) OP y ] where OP is CondKind
+// condition.
+class HVecPredWhile final : public HVecPredSetOperation {
+ public:
+ enum class CondKind {
+ kLE, // signed less than or equal.
+ kLO, // unsigned lower.
+ kLS, // unsigned lower or same.
+ kLT, // signed less.
+ kLast = kLT,
+ };
+
+ HVecPredWhile(ArenaAllocator* allocator,
+ HInstruction* left,
+ HInstruction* right,
+ CondKind cond,
+ DataType::Type packed_type,
+ size_t vector_length,
+ uint32_t dex_pc) :
+ HVecPredSetOperation(kVecPredWhile,
+ allocator,
+ packed_type,
+ SideEffects::None(),
+ /* number_of_inputs= */ 2,
+ vector_length,
+ dex_pc) {
+ DCHECK(!left->IsVecOperation());
+ DCHECK(!left->IsVecPredSetOperation());
+ DCHECK(!right->IsVecOperation());
+ DCHECK(!right->IsVecPredSetOperation());
+ DCHECK(DataType::IsIntegralType(left->GetType()));
+ DCHECK(DataType::IsIntegralType(right->GetType()));
+ SetRawInputAt(0, left);
+ SetRawInputAt(1, right);
+ SetPackedField<CondKindField>(cond);
+ }
+
+ // This is a special loop control instruction which must not be predicated.
+ bool MustBePredicatedInPredicatedSIMDMode() override { return false; }
+
+ CondKind GetCondKind() const {
+ return GetPackedField<CondKindField>();
+ }
+
+ DECLARE_INSTRUCTION(VecPredWhile);
+
+ protected:
+ // Additional packed bits.
+ static constexpr size_t kCondKind = HVecOperation::kNumberOfVectorOpPackedBits;
+ static constexpr size_t kCondKindSize =
+ MinimumBitsToStore(static_cast<size_t>(CondKind::kLast));
+ static constexpr size_t kNumberOfVecPredConditionPackedBits = kCondKind + kCondKindSize;
+ static_assert(kNumberOfVecPredConditionPackedBits <= kMaxNumberOfPackedBits,
+ "Too many packed fields.");
+ using CondKindField = BitField<CondKind, kCondKind, kCondKindSize>;
+
+ DEFAULT_COPY_CONSTRUCTOR(VecPredWhile);
+};
+
+// Evaluates the predicate condition (PCondKind) for a vector predicate; outputs
+// a scalar boolean value result.
+//
+// Note: as VecPredCondition can be also predicated, only active elements (determined by the
+// instruction's governing predicate) of the input vector predicate are used for condition
+// evaluation.
+//
+// Note: this instruction is currently used as a workaround for the fact that IR instructions
+// can't have more than one output.
+class HVecPredCondition final : public HVecOperation {
+ public:
+ // To get more info on the condition kinds please see "2.2 Process state, PSTATE" section of
+ // "ARM Architecture Reference Manual Supplement. The Scalable Vector Extension (SVE),
+ // for ARMv8-A".
+ enum class PCondKind {
+ kNone, // No active elements were TRUE.
+ kAny, // An active element was TRUE.
+ kNLast, // The last active element was not TRUE.
+ kLast, // The last active element was TRUE.
+ kFirst, // The first active element was TRUE.
+ kNFirst, // The first active element was not TRUE.
+ kPMore, // An active element was TRUE but not the last active element.
+ kPLast, // The last active element was TRUE or no active elements were TRUE.
+ kEnumLast = kPLast
+ };
+
+ HVecPredCondition(ArenaAllocator* allocator,
+ HInstruction* input,
+ PCondKind pred_cond,
+ DataType::Type packed_type,
+ size_t vector_length,
+ uint32_t dex_pc)
+ : HVecOperation(kVecPredCondition,
+ allocator,
+ packed_type,
+ SideEffects::None(),
+ /* number_of_inputs */ 1,
+ vector_length,
+ dex_pc) {
+ DCHECK(input->IsVecPredSetOperation());
+ SetRawInputAt(0, input);
+ // Overrides the kSIMDType set by the VecOperation constructor.
+ SetPackedField<TypeField>(DataType::Type::kBool);
+ SetPackedField<CondKindField>(pred_cond);
+ }
+
+ // This instruction is currently used only as a special loop control instruction
+ // which must not be predicated.
+ // TODO: Remove the constraint.
+ bool MustBePredicatedInPredicatedSIMDMode() override { return false; }
+
+ PCondKind GetPCondKind() const {
+ return GetPackedField<CondKindField>();
+ }
+
+ DECLARE_INSTRUCTION(VecPredCondition);
+
+ protected:
+ // Additional packed bits.
+ static constexpr size_t kCondKind = HVecOperation::kNumberOfVectorOpPackedBits;
+ static constexpr size_t kCondKindSize =
+ MinimumBitsToStore(static_cast<size_t>(PCondKind::kEnumLast));
+ static constexpr size_t kNumberOfVecPredConditionPackedBits = kCondKind + kCondKindSize;
+ static_assert(kNumberOfVecPredConditionPackedBits <= kMaxNumberOfPackedBits,
+ "Too many packed fields.");
+ using CondKindField = BitField<PCondKind, kCondKind, kCondKindSize>;
+
+ DEFAULT_COPY_CONSTRUCTOR(VecPredCondition);
+};
+
} // namespace art
#endif // ART_COMPILER_OPTIMIZING_NODES_VECTOR_H_
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index 02751cb..45d31ba 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -118,6 +118,7 @@
visualizer_output_(visualizer_output),
visualizer_enabled_(!compiler_options.GetDumpCfgFileName().empty()),
visualizer_(&visualizer_oss_, graph, *codegen),
+ codegen_(codegen),
visualizer_dump_mutex_(dump_mutex),
graph_in_bad_state_(false) {
if (timing_logger_enabled_ || visualizer_enabled_) {
@@ -190,7 +191,7 @@
// Validate the HGraph if running in debug mode.
if (kIsDebugBuild) {
if (!graph_in_bad_state_) {
- GraphChecker checker(graph_);
+ GraphChecker checker(graph_, codegen_);
last_seen_graph_size_ = checker.Run(pass_change, last_seen_graph_size_);
if (!checker.IsValid()) {
LOG(FATAL) << "Error after " << pass_name << ": " << Dumpable<GraphChecker>(checker);
@@ -230,6 +231,7 @@
std::ostream* visualizer_output_;
bool visualizer_enabled_;
HGraphVisualizer visualizer_;
+ CodeGenerator* codegen_;
Mutex& visualizer_dump_mutex_;
// Flag to be set by the compiler if the pass failed and the graph is not