From 740ae3479b54495d2dda92a000107325b08faf35 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Mon, 3 Jun 2024 16:25:38 +0100 Subject: Support all conditions in predicated vectorization Support all condition types inside the condition when performing diamond loop auto-vectorization. This allows diamond loop auto-vectorization to be performed on a greater variety of loops. To support this change, new vector condition nodes are added to mirror the scalar condition nodes. Also add a new gtest class to test whether predicated vectorization can be performed on different combinations of condition types and data types. Authors: Chris Jones , Konstantin Baladurin Test: export ART_FORCE_TRY_PREDICATED_SIMD=true && \ art/test.py --target --optimizing Test: art/test.py --target --host --optimizing Test: 661-checker-simd-cf-loops Test: art/test.py --gtest art_compiler_tests Change-Id: Ic9c925f1a58ada13d9031de3b445dcd4f77764b7 --- compiler/optimizing/nodes_vector.h | 277 +++++++++++++++++++++++++++++++++++-- 1 file changed, 264 insertions(+), 13 deletions(-) (limited to 'compiler/optimizing/nodes_vector.h') diff --git a/compiler/optimizing/nodes_vector.h b/compiler/optimizing/nodes_vector.h index a5f5720f7e..a4c22f7017 100644 --- a/compiler/optimizing/nodes_vector.h +++ b/compiler/optimizing/nodes_vector.h @@ -1467,19 +1467,16 @@ class HVecPredToBoolean final : public HVecOperation { // // viz. [ p1, .. , pn ] = [ x1 OP y1 , x2 OP y2, .. , xn OP yn] where OP is CondKind // condition. -// -// Currently only kEqual is supported by this vector instruction - we don't even define -// the kCondType here. -// TODO: support other condition ops. -class HVecCondition final : public HVecPredSetOperation { +class HVecCondition : public HVecPredSetOperation { public: - HVecCondition(ArenaAllocator* allocator, + HVecCondition(InstructionKind kind, + ArenaAllocator* allocator, HInstruction* left, HInstruction* right, DataType::Type packed_type, size_t vector_length, - uint32_t dex_pc) : - HVecPredSetOperation(kVecCondition, + uint32_t dex_pc = kNoDexPc) : + HVecPredSetOperation(kind, allocator, packed_type, SideEffects::None(), @@ -1494,22 +1491,276 @@ class HVecCondition final : public HVecPredSetOperation { SetRawInputAt(1, right); } - DECLARE_INSTRUCTION(VecCondition); + DECLARE_ABSTRACT_INSTRUCTION(VecCondition); + + virtual IfCondition GetCondition() const = 0; + + static HVecCondition* Create(HGraph* graph, + IfCondition cond, + HInstruction* lhs, + HInstruction* rhs, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc); protected: DEFAULT_COPY_CONSTRUCTOR(VecCondition); }; +// Instruction to check if two vector inputs are equal to each other. +class HVecEqual final : public HVecCondition { + public: + HVecEqual(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecEqual, allocator, left, right, packed_type, vector_length, dex_pc) {} + + DECLARE_INSTRUCTION(VecEqual); + + IfCondition GetCondition() const override { + return kCondEQ; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecEqual); +}; + +// Instruction to check if two vector inputs are not equal to each other. +class HVecNotEqual final : public HVecCondition { + public: + HVecNotEqual(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecNotEqual, allocator, left, right, packed_type, vector_length, dex_pc) {} + + DECLARE_INSTRUCTION(VecNotEqual); + + IfCondition GetCondition() const override { + return kCondNE; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecNotEqual); +}; + +// Instruction to check if one vector input is less than the other. +class HVecLessThan final : public HVecCondition { + public: + HVecLessThan(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecLessThan, allocator, left, right, packed_type, vector_length, dex_pc) {} + + DECLARE_INSTRUCTION(VecLessThan); + + IfCondition GetCondition() const override { + return kCondLT; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecLessThan); +}; + +// Instruction to check if one vector input is less than or equal to the other. +class HVecLessThanOrEqual final : public HVecCondition { + public: + HVecLessThanOrEqual(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecLessThanOrEqual, + allocator, + left, + right, + packed_type, + vector_length, + dex_pc) {} + + DECLARE_INSTRUCTION(VecLessThanOrEqual); + + IfCondition GetCondition() const override { + return kCondLE; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecLessThanOrEqual); +}; + +// Instruction to check if one vector input is greater than the other. +class HVecGreaterThan final : public HVecCondition { + public: + HVecGreaterThan(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecGreaterThan, + allocator, + left, + right, + packed_type, + vector_length, + dex_pc) {} + + DECLARE_INSTRUCTION(VecGreaterThan); + + IfCondition GetCondition() const override { + return kCondGT; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecGreaterThan); +}; + +// Instruction to check if one vector input is greater than or equal to the other. +class HVecGreaterThanOrEqual final : public HVecCondition { + public: + HVecGreaterThanOrEqual(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecGreaterThanOrEqual, + allocator, + left, + right, + packed_type, + vector_length, + dex_pc) {} + + DECLARE_INSTRUCTION(VecGreaterThanOrEqual); + + IfCondition GetCondition() const override { + return kCondGE; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecGreaterThanOrEqual); +}; + +// Instruction to check if one unsigned vector input is less than the other, using unsigned +// comparison. +class HVecBelow final : public HVecCondition { + public: + HVecBelow(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecBelow, allocator, left, right, packed_type, vector_length, dex_pc) {} + + DECLARE_INSTRUCTION(VecBelow); + + IfCondition GetCondition() const override { + return kCondB; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecBelow); +}; + +// Instruction to check if one unsigned vector input is less than or equal to the other, using +// unsigned comparison. +class HVecBelowOrEqual final : public HVecCondition { + public: + HVecBelowOrEqual(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecBelowOrEqual, + allocator, + left, + right, + packed_type, + vector_length, + dex_pc) {} + + DECLARE_INSTRUCTION(VecBelowOrEqual); + + IfCondition GetCondition() const override { + return kCondBE; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecBelowOrEqual); +}; + +// Instruction to check if one unsigned vector input is greater than the other, using unsigned +// comparison. +class HVecAbove final : public HVecCondition { + public: + HVecAbove(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecAbove, allocator, left, right, packed_type, vector_length, dex_pc) {} + + DECLARE_INSTRUCTION(VecAbove); + + IfCondition GetCondition() const override { + return kCondA; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecAbove); +}; + +// Instruction to check if one unsigned vector input is greater than or equal to the other, using +// unsigned comparison. +class HVecAboveOrEqual final : public HVecCondition { + public: + HVecAboveOrEqual(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecCondition(kVecAboveOrEqual, + allocator, + left, + right, + packed_type, + vector_length, + dex_pc) {} + + DECLARE_INSTRUCTION(VecAboveOrEqual); + + IfCondition GetCondition() const override { + return kCondAE; + } + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecAboveOrEqual); +}; + // Inverts every component in the predicate vector. // // viz. [ p1, .. , pn ] = [ !px1 , !px2 , .. , !pxn ]. class HVecPredNot final : public HVecPredSetOperation { public: HVecPredNot(ArenaAllocator* allocator, - HInstruction* input, - DataType::Type packed_type, - size_t vector_length, - uint32_t dex_pc) : + HInstruction* input, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc) : HVecPredSetOperation(kVecPredNot, allocator, packed_type, -- cgit v1.2.3-59-g8ed1b