summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author Vladimir Marko <vmarko@google.com> 2024-06-13 16:16:43 +0200
committer VladimĂ­r Marko <vmarko@google.com> 2025-03-20 03:54:03 -0700
commit2fda8f36c8ccb34a49c31d76be4eaaa6d9d45760 (patch)
tree2af072e1dae120cf6b00d29deb54e0104c8d9b12
parent59aaec4d51436995ace22c4bd6f35ff7f6f8aaf4 (diff)
Fix LSE to track type conversions correctly.
Add a new state to LSE's `Value` class to mark loads that depend on loop Phi placeholders but may require a type conversion. We store the load in the `Value` and use the `loads_requiring_loop_phi_[load->GetId()]->value` to form a singly-linked list leading to the actual Phi placeholder. During `LSEVisitor::ProcessLoadsRequiringLoopPhis()`, we may replace these records, so a walk over the singly-linked list in that stage can terminate with a replacement instruction instead of the loop Phi placeholder. Test: m test-art-host-gtest Test: testrunner.py --host --optimizing Bug: 341476044 Change-Id: Id8f2edf4dfaa82dc3f25ddf47f3fe1da8f2fb3ea
-rw-r--r--compiler/optimizing/load_store_elimination.cc396
-rw-r--r--compiler/optimizing/load_store_elimination_test.cc126
-rw-r--r--test/530-checker-lse/src/Main.java2
-rw-r--r--test/530-checker-lse/src/TypeConversions.java276
4 files changed, 722 insertions, 78 deletions
diff --git a/compiler/optimizing/load_store_elimination.cc b/compiler/optimizing/load_store_elimination.cc
index efe6253f96..799c1dcbf0 100644
--- a/compiler/optimizing/load_store_elimination.cc
+++ b/compiler/optimizing/load_store_elimination.cc
@@ -24,6 +24,7 @@
#include "base/arena_allocator.h"
#include "base/arena_bit_vector.h"
#include "base/array_ref.h"
+#include "base/bit_utils_iterator.h"
#include "base/bit_vector-inl.h"
#include "base/bit_vector.h"
#include "base/globals.h"
@@ -304,9 +305,12 @@ class LSEVisitor final : private HGraphDelegateVisitor {
struct NeedsNonLoopPhiMarker {
PhiPlaceholder phi_;
};
- struct NeedsLoopPhiMarker {
+ struct NeedsPlainLoopPhiMarker {
PhiPlaceholder phi_;
};
+ struct NeedsConvertedLoopPhiMarker {
+ HInstruction* load_; // Load from a narrower location than the loop phi it needs.
+ };
static constexpr Value Invalid() {
return Value(ValuelessType::kInvalid);
@@ -334,12 +338,16 @@ class LSEVisitor final : private HGraphDelegateVisitor {
return Value(NeedsNonLoopPhiMarker{phi_placeholder});
}
- static constexpr Value ForLoopPhiPlaceholder(PhiPlaceholder phi_placeholder) {
- return Value(NeedsLoopPhiMarker{phi_placeholder});
+ static constexpr Value ForPlainLoopPhiPlaceholder(PhiPlaceholder phi_placeholder) {
+ return Value(NeedsPlainLoopPhiMarker{phi_placeholder});
+ }
+
+ static constexpr Value ForConvertedLoopPhiPlaceholder(HInstruction* load) {
+ return Value(NeedsConvertedLoopPhiMarker{load});
}
static constexpr Value ForPhiPlaceholder(PhiPlaceholder phi_placeholder, bool needs_loop_phi) {
- return needs_loop_phi ? ForLoopPhiPlaceholder(phi_placeholder)
+ return needs_loop_phi ? ForPlainLoopPhiPlaceholder(phi_placeholder)
: ForNonLoopPhiPlaceholder(phi_placeholder);
}
@@ -370,8 +378,16 @@ class LSEVisitor final : private HGraphDelegateVisitor {
return std::holds_alternative<NeedsNonLoopPhiMarker>(value_);
}
+ bool NeedsPlainLoopPhi() const {
+ return std::holds_alternative<NeedsPlainLoopPhiMarker>(value_);
+ }
+
+ bool NeedsConvertedLoopPhi() const {
+ return std::holds_alternative<NeedsConvertedLoopPhiMarker>(value_);
+ }
+
bool NeedsLoopPhi() const {
- return std::holds_alternative<NeedsLoopPhiMarker>(value_);
+ return NeedsPlainLoopPhi() || NeedsConvertedLoopPhi();
}
bool NeedsPhi() const {
@@ -384,11 +400,11 @@ class LSEVisitor final : private HGraphDelegateVisitor {
}
PhiPlaceholder GetPhiPlaceholder() const {
- DCHECK(NeedsPhi());
if (NeedsNonLoopPhi()) {
return std::get<NeedsNonLoopPhiMarker>(value_).phi_;
} else {
- return std::get<NeedsLoopPhiMarker>(value_).phi_;
+ DCHECK(NeedsPlainLoopPhi());
+ return std::get<NeedsPlainLoopPhiMarker>(value_).phi_;
}
}
@@ -397,6 +413,11 @@ class LSEVisitor final : private HGraphDelegateVisitor {
return GetPhiPlaceholder().GetHeapLocation();
}
+ HInstruction* GetLoopPhiConversionLoad() const {
+ DCHECK(NeedsConvertedLoopPhi());
+ return std::get<NeedsConvertedLoopPhiMarker>(value_).load_;
+ }
+
constexpr bool ExactEquals(Value other) const;
constexpr bool Equals(Value other) const;
@@ -414,7 +435,8 @@ class LSEVisitor final : private HGraphDelegateVisitor {
using ValueHolder = std::variant<ValuelessType,
HInstruction*,
NeedsNonLoopPhiMarker,
- NeedsLoopPhiMarker>;
+ NeedsPlainLoopPhiMarker,
+ NeedsConvertedLoopPhiMarker>;
constexpr ValuelessType GetValuelessType() const {
return std::get<ValuelessType>(value_);
}
@@ -428,11 +450,68 @@ class LSEVisitor final : private HGraphDelegateVisitor {
static_assert(std::is_move_assignable<PhiPlaceholder>::value);
};
- friend constexpr bool operator==(const Value::NeedsLoopPhiMarker& p1,
- const Value::NeedsLoopPhiMarker& p2);
+ friend constexpr bool operator==(const Value::NeedsPlainLoopPhiMarker& p1,
+ const Value::NeedsPlainLoopPhiMarker& p2);
+ friend constexpr bool operator==(const Value::NeedsConvertedLoopPhiMarker& p1,
+ const Value::NeedsConvertedLoopPhiMarker& p2);
friend constexpr bool operator==(const Value::NeedsNonLoopPhiMarker& p1,
const Value::NeedsNonLoopPhiMarker& p2);
+ class TypeConversionSet {
+ public:
+ TypeConversionSet() : type_conversions_(0u) {}
+
+ void Add(DataType::Type result_type) {
+ static_assert(enum_cast<>(DataType::Type::kLast) < BitSizeOf<uint32_t>());
+ type_conversions_ |= 1u << enum_cast<>(result_type);
+ }
+
+ void Add(TypeConversionSet other) {
+ type_conversions_ |= other.type_conversions_;
+ }
+
+ bool AreAllTypeConversionsImplicit(HInstruction* input) const {
+ if (type_conversions_ != 0u) {
+ if (input->IsIntConstant()) {
+ int32_t value = input->AsIntConstant()->GetValue();
+ for (uint32_t raw_type : LowToHighBits(type_conversions_)) {
+ DataType::Type type = enum_cast<DataType::Type>(raw_type);
+ if (!DataType::IsTypeConversionImplicit(value, type)) {
+ return false;
+ }
+ }
+ } else {
+ DataType::Type input_type = input->GetType();
+ for (uint32_t raw_type : LowToHighBits(type_conversions_)) {
+ DataType::Type type = enum_cast<DataType::Type>(raw_type);
+ if (!DataType::IsTypeConversionImplicit(input_type, type)) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ private:
+ uint32_t type_conversions_;
+ };
+
+ Value SkipTypeConversions(Value value,
+ /*inout*/ TypeConversionSet* type_conversions = nullptr) const {
+ while (value.NeedsConvertedLoopPhi()) {
+ HInstruction* conversion_load = value.GetLoopPhiConversionLoad();
+ DCHECK(!conversion_load->IsVecLoad());
+ if (type_conversions != nullptr) {
+ type_conversions->Add(conversion_load->GetType());
+ }
+ ValueRecord* prev_record = loads_requiring_loop_phi_[conversion_load->GetId()];
+ DCHECK(prev_record != nullptr);
+ value = prev_record->value;
+ }
+ return value;
+ }
+
// Get Phi placeholder index for access to `phi_placeholder_replacements_`
// and "visited" bit vectors during depth-first searches.
size_t PhiPlaceholderIndex(PhiPlaceholder phi_placeholder) const {
@@ -446,7 +525,7 @@ class LSEVisitor final : private HGraphDelegateVisitor {
}
size_t PhiPlaceholderIndex(Value phi_placeholder) const {
- return PhiPlaceholderIndex(phi_placeholder.GetPhiPlaceholder());
+ return PhiPlaceholderIndex(SkipTypeConversions(phi_placeholder).GetPhiPlaceholder());
}
bool IsEscapingObject(ReferenceInfo* info) { return !info->IsSingletonAndRemovable(); }
@@ -466,7 +545,8 @@ class LSEVisitor final : private HGraphDelegateVisitor {
}
Value Replacement(Value value) const {
- DCHECK(value.NeedsPhi()) << value << " phase: " << current_phase_;
+ DCHECK(value.NeedsNonLoopPhi() || value.NeedsPlainLoopPhi())
+ << value << " phase: " << current_phase_;
Value replacement = phi_placeholder_replacements_[PhiPlaceholderIndex(value)];
DCHECK(replacement.IsUnknown() || replacement.IsInstruction());
DCHECK(replacement.IsUnknown() ||
@@ -475,7 +555,12 @@ class LSEVisitor final : private HGraphDelegateVisitor {
}
Value ReplacementOrValue(Value value) const {
- if (value.NeedsPhi() && phi_placeholder_replacements_[PhiPlaceholderIndex(value)].IsValid()) {
+ if (value.NeedsConvertedLoopPhi() &&
+ substitute_instructions_for_loads_[value.GetLoopPhiConversionLoad()->GetId()] != nullptr) {
+ return Value::ForInstruction(
+ substitute_instructions_for_loads_[value.GetLoopPhiConversionLoad()->GetId()]);
+ } else if ((value.NeedsNonLoopPhi() || value.NeedsPlainLoopPhi()) &&
+ phi_placeholder_replacements_[PhiPlaceholderIndex(value)].IsValid()) {
return Replacement(value);
} else {
DCHECK_IMPLIES(value.IsInstruction(),
@@ -490,6 +575,66 @@ class LSEVisitor final : private HGraphDelegateVisitor {
Value stored_by;
};
+ // Calculate the value stored in location `idx` for a loop Phi placeholder-dependent `load`.
+ Value StoredValueForLoopPhiPlaceholderDependentLoad(size_t idx, HInstruction* load) const {
+ DCHECK(IsLoad(load));
+ DCHECK_LT(static_cast<size_t>(load->GetId()), loads_requiring_loop_phi_.size());
+ DCHECK(loads_requiring_loop_phi_[load->GetId()] != nullptr);
+ Value loaded_value = loads_requiring_loop_phi_[load->GetId()]->value;
+ DCHECK(loaded_value.NeedsLoopPhi());
+ DataType::Type load_type = load->GetType();
+ size_t load_size = DataType::Size(load_type);
+ size_t store_size = DataType::Size(heap_location_collector_.GetHeapLocation(idx)->GetType());
+
+ if (kIsDebugBuild && load->IsVecLoad()) {
+ // For vector operations, the load type is always `Float64` and therefore the store size is
+ // never higher and we do not record any conversions below. This is OK because we currently
+ // do not vectorize any loops with widening operations.
+ CHECK_EQ(load_size, DataType::Size(DataType::Type::kFloat64));
+ CHECK_LE(store_size, load_size);
+ CHECK(!loaded_value.NeedsConvertedLoopPhi());
+ } else if (kIsDebugBuild) {
+ // There are no implicit conversions between 64-bit types and smaller types.
+ // We shall not record any conversions for 64-bit types.
+ CHECK_EQ(load_size == DataType::Size(DataType::Type::kInt64),
+ store_size == DataType::Size(DataType::Type::kInt64));
+ CHECK_IMPLIES(load_size == DataType::Size(DataType::Type::kInt64),
+ !loaded_value.NeedsConvertedLoopPhi());
+ }
+ // The `loaded_value` can record a conversion only if the `load` was from
+ // a wider field than the previous converting load.
+ DCHECK_IMPLIES(loaded_value.NeedsConvertedLoopPhi(),
+ load_size > DataType::Size(loaded_value.GetLoopPhiConversionLoad()->GetType()));
+
+ Value value = loaded_value;
+ if (load_size < store_size) {
+ // Add a type conversion to a narrow type unless it's an implicit conversion
+ // from an already converted value.
+ if (!loaded_value.NeedsConvertedLoopPhi() ||
+ !DataType::IsTypeConversionImplicit(loaded_value.GetLoopPhiConversionLoad()->GetType(),
+ load_type)) {
+ value = Value::ForConvertedLoopPhiPlaceholder(load);
+ } else {
+ DCHECK(value.Equals(loaded_value));
+ }
+ } else {
+ // Remove conversions to types at least as wide as the field we're storing to.
+ // We record only conversions that define sign-/zero-extension bits to store.
+ while (value.NeedsConvertedLoopPhi() &&
+ DataType::Size(value.GetLoopPhiConversionLoad()->GetType()) >= store_size) {
+ HInstruction* conversion_load = value.GetLoopPhiConversionLoad();
+ ValueRecord* prev_record =
+ loads_requiring_loop_phi_[value.GetLoopPhiConversionLoad()->GetId()];
+ DCHECK(prev_record != nullptr);
+ value = prev_record->value;
+ DCHECK(value.NeedsLoopPhi());
+ }
+ }
+
+ DCHECK_EQ(PhiPlaceholderIndex(loaded_value), PhiPlaceholderIndex(value));
+ return value;
+ }
+
HTypeConversion* FindOrAddTypeConversionIfNecessary(HInstruction* instruction,
HInstruction* value,
DataType::Type expected_type) {
@@ -502,6 +647,10 @@ class LSEVisitor final : private HGraphDelegateVisitor {
return nullptr;
}
+ // All vector instructions report their type as `Float64`, so the conversion is implicit.
+ // This is OK because we currently do not vectorize any loops with widening operations.
+ DCHECK(!instruction->IsVecLoad());
+
// Check if there is already a suitable TypeConversion we can reuse.
for (const HUseListNode<HInstruction*>& use : value->GetUses()) {
if (use.GetUser()->IsTypeConversion() &&
@@ -716,6 +865,7 @@ class LSEVisitor final : private HGraphDelegateVisitor {
/*out*/ ArenaBitVector* phi_placeholders_to_materialize,
DataType::Type type,
bool can_use_default_or_phi);
+ void MaterializeTypeConversionsIfNeeded(Value value);
bool MaterializeLoopPhis(const ScopedArenaVector<size_t>& phi_placeholder_indexes,
DataType::Type type);
bool MaterializeLoopPhis(ArrayRef<const size_t> phi_placeholder_indexes, DataType::Type type);
@@ -1211,11 +1361,16 @@ constexpr bool operator==(const LSEVisitor::PhiPlaceholder& p1,
return p1.Equals(p2);
}
-constexpr bool operator==(const LSEVisitor::Value::NeedsLoopPhiMarker& p1,
- const LSEVisitor::Value::NeedsLoopPhiMarker& p2) {
+constexpr bool operator==(const LSEVisitor::Value::NeedsPlainLoopPhiMarker& p1,
+ const LSEVisitor::Value::NeedsPlainLoopPhiMarker& p2) {
return p1.phi_ == p2.phi_;
}
+constexpr bool operator==(const LSEVisitor::Value::NeedsConvertedLoopPhiMarker& p1,
+ const LSEVisitor::Value::NeedsConvertedLoopPhiMarker& p2) {
+ return p1.load_ == p2.load_;
+}
+
constexpr bool operator==(const LSEVisitor::Value::NeedsNonLoopPhiMarker& p1,
const LSEVisitor::Value::NeedsNonLoopPhiMarker& p2) {
return p1.phi_ == p2.phi_;
@@ -1257,9 +1412,12 @@ std::ostream& LSEVisitor::Value::Dump(std::ostream& os) const {
} else if (IsInstruction()) {
return os << "Instruction[id: " << GetInstruction()->GetId()
<< ", block: " << GetInstruction()->GetBlock()->GetBlockId() << "]";
- } else if (NeedsLoopPhi()) {
- return os << "NeedsLoopPhi[block: " << GetPhiPlaceholder().GetBlockId()
+ } else if (NeedsPlainLoopPhi()) {
+ return os << "NeedsPlainLoopPhi[block: " << GetPhiPlaceholder().GetBlockId()
<< ", heap_loc: " << GetPhiPlaceholder().GetHeapLocation() << "]";
+ } else if (NeedsConvertedLoopPhi()) {
+ return os << "NeedsConvertedLoopPhi[id: " << GetLoopPhiConversionLoad()->GetId()
+ << ", block: " << GetLoopPhiConversionLoad()->GetBlock()->GetBlockId() << "]";
} else {
return os << "NeedsNonLoopPhi[block: " << GetPhiPlaceholder().GetBlockId()
<< ", heap_loc: " << GetPhiPlaceholder().GetHeapLocation() << "]";
@@ -1332,7 +1490,7 @@ LSEVisitor::Value LSEVisitor::PrepareLoopValue(HBasicBlock* block, size_t idx) {
}
}
PhiPlaceholder phi_placeholder = GetPhiPlaceholder(block->GetBlockId(), idx);
- return ReplacementOrValue(Value::ForLoopPhiPlaceholder(phi_placeholder));
+ return ReplacementOrValue(Value::ForPlainLoopPhiPlaceholder(phi_placeholder));
}
LSEVisitor::Value LSEVisitor::PrepareLoopStoredBy(HBasicBlock* block, size_t idx) {
@@ -1350,7 +1508,7 @@ LSEVisitor::Value LSEVisitor::PrepareLoopStoredBy(HBasicBlock* block, size_t idx
return Value::Unknown();
}
PhiPlaceholder phi_placeholder = GetPhiPlaceholder(block->GetBlockId(), idx);
- return Value::ForLoopPhiPlaceholder(phi_placeholder);
+ return Value::ForPlainLoopPhiPlaceholder(phi_placeholder);
}
void LSEVisitor::PrepareLoopRecords(HBasicBlock* block) {
@@ -1370,7 +1528,7 @@ void LSEVisitor::PrepareLoopRecords(HBasicBlock* block) {
{/*value=*/Value::Unknown(), /*stored_by=*/Value::Unknown()});
// Also keep the stores before the loop header, including in blocks that were not visited yet.
for (size_t idx = 0u; idx != num_heap_locations; ++idx) {
- KeepStores(Value::ForLoopPhiPlaceholder(GetPhiPlaceholder(block->GetBlockId(), idx)));
+ KeepStores(Value::ForPlainLoopPhiPlaceholder(GetPhiPlaceholder(block->GetBlockId(), idx)));
}
return;
}
@@ -1547,6 +1705,9 @@ void LSEVisitor::MaterializeNonLoopPhis(PhiPlaceholder phi_placeholder, DataType
void LSEVisitor::VisitGetLocation(HInstruction* instruction, size_t idx) {
DCHECK_NE(idx, HeapLocationCollector::kHeapLocationNotFound);
+ DCHECK_EQ(DataType::Size(heap_location_collector_.GetHeapLocation(idx)->GetType()),
+ DataType::Size(instruction->IsVecLoad() ? instruction->AsVecLoad()->GetPackedType()
+ : instruction->GetType()));
uint32_t block_id = instruction->GetBlock()->GetBlockId();
ScopedArenaVector<ValueRecord>& heap_values = heap_values_for_[block_id];
ValueRecord& record = heap_values[idx];
@@ -1590,7 +1751,7 @@ void LSEVisitor::VisitGetLocation(HInstruction* instruction, size_t idx) {
void LSEVisitor::VisitSetLocation(HInstruction* instruction, size_t idx, HInstruction* value) {
DCHECK_NE(idx, HeapLocationCollector::kHeapLocationNotFound);
DCHECK(!IsStore(value)) << value->DebugName();
- // value may already have a substitute.
+ // The `value` may already have a substitute.
value = FindSubstitute(value);
HBasicBlock* block = instruction->GetBlock();
ScopedArenaVector<ValueRecord>& heap_values = heap_values_for_[block->GetBlockId()];
@@ -1598,6 +1759,19 @@ void LSEVisitor::VisitSetLocation(HInstruction* instruction, size_t idx, HInstru
DCHECK_IMPLIES(record.value.IsInstruction(),
FindSubstitute(record.value.GetInstruction()) == record.value.GetInstruction());
+ // Calculate the new `Value` to store to the `record`.
+ Value new_value = Value::ForInstruction(value);
+ // Note that the `value` can be a newly created `Phi` with an id that falls outside
+ // the allocated `loads_requiring_loop_phi_` range.
+ DCHECK_IMPLIES(IsLoad(value) && !loads_requiring_loop_phi_.empty(),
+ static_cast<size_t>(value->GetId()) < loads_requiring_loop_phi_.size());
+ if (static_cast<size_t>(value->GetId()) < loads_requiring_loop_phi_.size() &&
+ loads_requiring_loop_phi_[value->GetId()] != nullptr) {
+ // Propapate the Phi placeholder or appropriate converting load to the record.
+ new_value = StoredValueForLoopPhiPlaceholderDependentLoad(idx, value);
+ DCHECK(new_value.NeedsLoopPhi());
+ }
+
if (record.value.Equals(value)) {
// Store into the heap location with the same value.
// This store can be eliminated right away.
@@ -1630,18 +1804,7 @@ void LSEVisitor::VisitSetLocation(HInstruction* instruction, size_t idx, HInstru
}
// Update the record.
- // Note that the `value` can be a newly created `Phi` with an id that falls outside
- // the allocated `loads_requiring_loop_phi_` range.
- DCHECK_IMPLIES(IsLoad(value) && !loads_requiring_loop_phi_.empty(),
- static_cast<size_t>(value->GetId()) < loads_requiring_loop_phi_.size());
- if (static_cast<size_t>(value->GetId()) < loads_requiring_loop_phi_.size() &&
- loads_requiring_loop_phi_[value->GetId()] != nullptr) {
- // Propapate the Phi placeholder to the record.
- record.value = loads_requiring_loop_phi_[value->GetId()]->value;
- DCHECK(record.value.NeedsLoopPhi());
- } else {
- record.value = Value::ForInstruction(value);
- }
+ record.value = new_value;
// Track the store in the value record. If the value is loaded or needed after
// return/deoptimization later, this store isn't really redundant.
record.stored_by = Value::ForInstruction(instruction);
@@ -1713,6 +1876,16 @@ bool LSEVisitor::TryReplacingLoopPhiPlaceholderWithDefault(
kArenaAllocLSE);
ScopedArenaVector<PhiPlaceholder> work_queue(allocator.Adapter(kArenaAllocLSE));
+ auto maybe_add_to_work_queue = [&](Value predecessor_value) {
+ // Visit the predecessor Phi placeholder if it's not visited yet.
+ DCHECK(predecessor_value.NeedsNonLoopPhi() || predecessor_value.NeedsPlainLoopPhi());
+ PhiPlaceholder predecessor_phi_placeholder = predecessor_value.GetPhiPlaceholder();
+ if (!visited.IsBitSet(PhiPlaceholderIndex(predecessor_phi_placeholder))) {
+ visited.SetBit(PhiPlaceholderIndex(predecessor_phi_placeholder));
+ work_queue.push_back(predecessor_phi_placeholder);
+ }
+ };
+
// Use depth first search to check if any non-Phi input is unknown.
const ArenaVector<HBasicBlock*>& blocks = GetGraph()->GetBlocks();
size_t num_heap_locations = heap_location_collector_.GetNumberOfHeapLocations();
@@ -1726,12 +1899,10 @@ bool LSEVisitor::TryReplacingLoopPhiPlaceholderWithDefault(
size_t idx = current_phi_placeholder.GetHeapLocation();
for (HBasicBlock* predecessor : block->GetPredecessors()) {
Value value = ReplacementOrValue(heap_values_for_[predecessor->GetBlockId()][idx].value);
+ // Skip over type conversions (these are unnecessary for the default value).
+ value = SkipTypeConversions(value);
if (value.NeedsPhi()) {
- // Visit the predecessor Phi placeholder if it's not visited yet.
- if (!visited.IsBitSet(PhiPlaceholderIndex(value))) {
- visited.SetBit(PhiPlaceholderIndex(value));
- work_queue.push_back(value.GetPhiPlaceholder());
- }
+ maybe_add_to_work_queue(value);
} else if (!value.Equals(Value::Default())) {
return false; // Report failure.
}
@@ -1754,19 +1925,18 @@ bool LSEVisitor::TryReplacingLoopPhiPlaceholderWithDefault(
// allow replacing the non-vector load of loop-invariant default values
// anyway, skip over paths that do not have any writes.
ValueRecord record = heap_values_for_[predecessor->GetBlockId()][i];
- while (record.stored_by.NeedsLoopPhi() &&
+ while (record.stored_by.NeedsPlainLoopPhi() &&
blocks[record.stored_by.GetPhiPlaceholder().GetBlockId()]->IsLoopHeader()) {
HLoopInformation* loop_info =
blocks[record.stored_by.GetPhiPlaceholder().GetBlockId()]->GetLoopInformation();
record = heap_values_for_[loop_info->GetPreHeader()->GetBlockId()][i];
}
+ DCHECK(!record.stored_by.NeedsConvertedLoopPhi());
Value value = ReplacementOrValue(record.value);
+ // Skip over type conversions (these are unnecessary for the default value).
+ value = SkipTypeConversions(value);
if (value.NeedsPhi()) {
- // Visit the predecessor Phi placeholder if it's not visited yet.
- if (!visited.IsBitSet(PhiPlaceholderIndex(value))) {
- visited.SetBit(PhiPlaceholderIndex(value));
- work_queue.push_back(value.GetPhiPlaceholder());
- }
+ maybe_add_to_work_queue(value);
} else if (!value.Equals(Value::Default())) {
return false; // Report failure.
}
@@ -1802,6 +1972,8 @@ bool LSEVisitor::TryReplacingLoopPhiPlaceholderWithSingleInput(
kArenaAllocLSE);
ScopedArenaVector<PhiPlaceholder> work_queue(allocator.Adapter(kArenaAllocLSE));
+ TypeConversionSet type_conversions;
+
// Use depth first search to check if any non-Phi input is unknown.
HInstruction* replacement = nullptr;
const ArenaVector<HBasicBlock*>& blocks = GetGraph()->GetBlocks();
@@ -1815,6 +1987,8 @@ bool LSEVisitor::TryReplacingLoopPhiPlaceholderWithSingleInput(
size_t idx = current_phi_placeholder.GetHeapLocation();
for (HBasicBlock* predecessor : current_block->GetPredecessors()) {
Value value = ReplacementOrValue(heap_values_for_[predecessor->GetBlockId()][idx].value);
+ // Skip type conversions but record them for checking later.
+ value = SkipTypeConversions(value, &type_conversions);
if (value.NeedsPhi()) {
// Visit the predecessor Phi placeholder if it's not visited yet.
if (!visited.IsBitSet(PhiPlaceholderIndex(value))) {
@@ -1837,8 +2011,16 @@ bool LSEVisitor::TryReplacingLoopPhiPlaceholderWithSingleInput(
// would invalidate the heap location in `VisitSetLocation()`.
}
- // Record replacement and report success.
+ // Check that there are no type conversions that would change the stored value.
DCHECK(replacement != nullptr);
+ if (!type_conversions.AreAllTypeConversionsImplicit(replacement)) {
+ return false;
+ }
+
+ // Record replacement and report success.
+ // Note: Replacements for the loads where we skipped type conversions above (and do not really
+ // need the type conversion) shall be recorded later, either when we process the loads in
+ // `ProcessLoadsRequiringLoopPhis()` or when needed to materialize another Phi.
for (uint32_t phi_placeholder_index : visited.Indexes()) {
DCHECK(phi_placeholder_replacements_[phi_placeholder_index].IsInvalid());
PhiPlaceholder curr = GetPhiPlaceholderAt(phi_placeholder_index);
@@ -1932,7 +2114,9 @@ std::optional<LSEVisitor::PhiPlaceholder> LSEVisitor::FindLoopPhisToMaterialize(
}
}
}
- if (value.NeedsLoopPhi()) {
+ // Skip type conversions. We're looking for the Phi placeholders now.
+ value = SkipTypeConversions(value);
+ if (value.NeedsPlainLoopPhi()) {
// Visit the predecessor Phi placeholder if it's not visited yet.
if (!phi_placeholders_to_materialize->IsBitSet(PhiPlaceholderIndex(value))) {
phi_placeholders_to_materialize->SetBit(PhiPlaceholderIndex(value));
@@ -1948,6 +2132,41 @@ std::optional<LSEVisitor::PhiPlaceholder> LSEVisitor::FindLoopPhisToMaterialize(
return std::nullopt;
}
+void LSEVisitor::MaterializeTypeConversionsIfNeeded(Value value) {
+ if (!value.NeedsConvertedLoopPhi()) {
+ return;
+ }
+ // There are at most 2 conversions (Uint8+Int16 or Int8+Uint16). Conversion to Int32
+ // is implicit and conversions to same or smaller size replace previous conversions.
+ static constexpr size_t kMaxConversionLoads = 2u;
+ HInstruction* conversion_loads[kMaxConversionLoads];
+ size_t num_conversion_loads = 0u;
+ do {
+ DCHECK_LT(num_conversion_loads, kMaxConversionLoads);
+ HInstruction* conversion_load = value.GetLoopPhiConversionLoad();
+ DCHECK(!conversion_load->IsVecLoad());
+ HInstruction* substitute = FindSubstitute(conversion_load);
+ if (substitute != conversion_load) {
+ value = Value::ForInstruction(substitute);
+ break;
+ }
+ conversion_loads[num_conversion_loads] = conversion_load;
+ ++num_conversion_loads;
+ ValueRecord* prev_record = loads_requiring_loop_phi_[conversion_load->GetId()];
+ DCHECK(prev_record != nullptr);
+ value = prev_record->value;
+ } while (value.NeedsConvertedLoopPhi());
+ value = value.NeedsPlainLoopPhi() ? Replacement(value) : value;
+ HInstruction* replacement = value.GetInstruction();
+ ArrayRef<HInstruction*> conversion_loads_array(conversion_loads, num_conversion_loads);
+ for (HInstruction* conversion_load : ReverseRange(conversion_loads_array)) {
+ AddRemovedLoad(conversion_load, replacement);
+ replacement = substitute_instructions_for_loads_[conversion_load->GetId()];
+ DCHECK(replacement != nullptr);
+ DCHECK(replacement->IsTypeConversion());
+ }
+}
+
bool LSEVisitor::MaterializeLoopPhis(const ScopedArenaVector<size_t>& phi_placeholder_indexes,
DataType::Type type) {
return MaterializeLoopPhis(ArrayRef<const size_t>(phi_placeholder_indexes), type);
@@ -1958,6 +2177,7 @@ bool LSEVisitor::MaterializeLoopPhis(ArrayRef<const size_t> phi_placeholder_inde
// Materialize all predecessors that do not need a loop Phi and determine if all inputs
// other than loop Phis are the same.
const ArenaVector<HBasicBlock*>& blocks = GetGraph()->GetBlocks();
+ TypeConversionSet type_conversions;
std::optional<Value> other_value = std::nullopt;
for (size_t phi_placeholder_index : phi_placeholder_indexes) {
PhiPlaceholder phi_placeholder = GetPhiPlaceholderAt(phi_placeholder_index);
@@ -1970,8 +2190,19 @@ bool LSEVisitor::MaterializeLoopPhis(ArrayRef<const size_t> phi_placeholder_inde
DCHECK(current_phase_ == Phase::kLoadElimination) << current_phase_;
MaterializeNonLoopPhis(value.GetPhiPlaceholder(), type);
value = Replacement(value);
+ } else if (value.NeedsConvertedLoopPhi()) {
+ TypeConversionSet local_type_conversions;
+ Value without_conversions = SkipTypeConversions(value, &local_type_conversions);
+ DCHECK(!without_conversions.NeedsNonLoopPhi()); // Would have been already materialized.
+ if (without_conversions.NeedsPlainLoopPhi()) {
+ type_conversions.Add(local_type_conversions);
+ value = without_conversions;
+ } else {
+ MaterializeTypeConversionsIfNeeded(value);
+ value = ReplacementOrValue(value);
+ }
}
- if (!value.NeedsLoopPhi()) {
+ if (!value.NeedsPlainLoopPhi()) {
if (!other_value) {
// The first other value we found.
other_value = value;
@@ -1986,9 +2217,13 @@ bool LSEVisitor::MaterializeLoopPhis(ArrayRef<const size_t> phi_placeholder_inde
}
DCHECK(other_value.has_value());
- if (!other_value->IsInvalid()) {
+ DCHECK(other_value->IsInvalid() || other_value->IsDefault() || other_value->IsInstruction());
+ if (other_value->IsDefault() || // Default value does not need type conversions.
+ (other_value->IsInstruction() &&
+ type_conversions.AreAllTypeConversionsImplicit(other_value->GetInstruction()))) {
HInstruction* replacement =
(other_value->IsDefault()) ? GetDefaultValue(type) : other_value->GetInstruction();
+ DCHECK(type_conversions.AreAllTypeConversionsImplicit(replacement));
for (size_t phi_placeholder_index : phi_placeholder_indexes) {
phi_placeholder_replacements_[phi_placeholder_index] = Value::ForInstruction(replacement);
}
@@ -2010,12 +2245,24 @@ bool LSEVisitor::MaterializeLoopPhis(ArrayRef<const size_t> phi_placeholder_inde
ArrayRef<HUserRecord<HInstruction*>> phi_inputs = phi->GetInputRecords();
auto cmp = [=, this](const HUserRecord<HInstruction*>& lhs, HBasicBlock* rhs) {
Value value = ReplacementOrValue(heap_values_for_[rhs->GetBlockId()][idx].value);
- if (value.NeedsPhi()) {
- DCHECK(value.GetPhiPlaceholder() == phi_placeholder);
- return lhs.GetInstruction() == phi;
+ HInstruction* lhs_instruction = lhs.GetInstruction();
+ while (value.NeedsConvertedLoopPhi()) {
+ HInstruction* conversion_load = value.GetLoopPhiConversionLoad();
+ if (!lhs_instruction->IsTypeConversion() ||
+ lhs_instruction->GetType() != conversion_load->GetType()) {
+ return false;
+ }
+ lhs_instruction = lhs_instruction->InputAt(0);
+ ValueRecord* prev_record = loads_requiring_loop_phi_[conversion_load->GetId()];
+ DCHECK(prev_record != nullptr);
+ value = prev_record->value;
+ }
+ if (value.NeedsPlainLoopPhi() && value.GetPhiPlaceholder().Equals(phi_placeholder)) {
+ return lhs_instruction == phi;
} else {
+ value = ReplacementOrValue(value);
DCHECK(value.IsDefault() || value.IsInstruction());
- return value.Equals(lhs.GetInstruction());
+ return value.Equals(lhs_instruction);
}
};
if (std::equal(phi_inputs.begin(), phi_inputs.end(), predecessors.begin(), cmp)) {
@@ -2049,7 +2296,9 @@ bool LSEVisitor::MaterializeLoopPhis(ArrayRef<const size_t> phi_placeholder_inde
<< "type=" << type << " vs phi-type=" << phi->GetType();
for (size_t i = 0, size = block->GetPredecessors().size(); i != size; ++i) {
HBasicBlock* predecessor = block->GetPredecessors()[i];
- Value value = ReplacementOrValue(heap_values_for_[predecessor->GetBlockId()][idx].value);
+ Value predecessor_value = heap_values_for_[predecessor->GetBlockId()][idx].value;
+ MaterializeTypeConversionsIfNeeded(predecessor_value);
+ Value value = ReplacementOrValue(predecessor_value);
HInstruction* input = value.IsDefault() ? GetDefaultValue(type) : value.GetInstruction();
DCHECK_NE(input->GetType(), DataType::Type::kVoid);
phi->SetRawInputAt(i, input);
@@ -2319,9 +2568,10 @@ void LSEVisitor::ProcessLoopPhiWithUnknownInput(PhiPlaceholder loop_phi_with_unk
// at the end of the block.
Value replacement = ReplacementOrValue(record->value);
if (replacement.NeedsLoopPhi()) {
- // No replacement yet, use the Phi placeholder from the load.
+ // No replacement yet. Use the Phi placeholder or an appropriate converting load.
DCHECK(record->value.NeedsLoopPhi());
- local_heap_values[idx] = record->value;
+ local_heap_values[idx] = StoredValueForLoopPhiPlaceholderDependentLoad(idx, stored_value);
+ DCHECK(local_heap_values[idx].NeedsLoopPhi());
} else {
// If the load fetched a known value, use it, otherwise use the load.
local_heap_values[idx] = Value::ForInstruction(
@@ -2401,12 +2651,17 @@ void LSEVisitor::ProcessLoadsRequiringLoopPhis() {
continue;
}
HInstruction* load = load_store_record.load_or_store;
- while (record->value.NeedsLoopPhi() &&
- phi_placeholder_replacements_[PhiPlaceholderIndex(record->value)].IsInvalid()) {
+ while (record->value.NeedsLoopPhi()) {
+ Value without_conversions = SkipTypeConversions(record->value);
+ if (!without_conversions.NeedsPlainLoopPhi() ||
+ phi_placeholder_replacements_[PhiPlaceholderIndex(without_conversions)].IsValid()) {
+ break;
+ }
std::optional<PhiPlaceholder> loop_phi_with_unknown_input =
- TryToMaterializeLoopPhis(record->value.GetPhiPlaceholder(), load);
- DCHECK_EQ(loop_phi_with_unknown_input.has_value(),
- phi_placeholder_replacements_[PhiPlaceholderIndex(record->value)].IsInvalid());
+ TryToMaterializeLoopPhis(without_conversions.GetPhiPlaceholder(), load);
+ DCHECK_EQ(
+ loop_phi_with_unknown_input.has_value(),
+ phi_placeholder_replacements_[PhiPlaceholderIndex(without_conversions)].IsInvalid());
if (loop_phi_with_unknown_input) {
DCHECK_GE(GetGraph()
->GetBlocks()[loop_phi_with_unknown_input->GetBlockId()]
@@ -2416,15 +2671,25 @@ void LSEVisitor::ProcessLoadsRequiringLoopPhis() {
ProcessLoopPhiWithUnknownInput(*loop_phi_with_unknown_input);
}
}
- // The load could have been marked as unreplaceable (and stores marked for keeping)
- // or marked for replacement with an instruction in ProcessLoopPhiWithUnknownInput().
+ // The load, or converting load's underlying phi placeholder, could have been marked
+ // as unreplaceable (and stores marked for keeping) or marked for replacement with an
+ // instruction in `ProcessLoopPhiWithUnknownInput()`.
DCHECK(record->value.IsUnknown() ||
record->value.IsInstruction() ||
record->value.NeedsLoopPhi());
if (record->value.NeedsLoopPhi()) {
- record->value = Replacement(record->value);
+ MaterializeTypeConversionsIfNeeded(record->value);
+ record->value = ReplacementOrValue(record->value);
HInstruction* heap_value = record->value.GetInstruction();
- AddRemovedLoad(load, heap_value);
+ // Type conversion substitutes can be created by `MaterializeTypeConversionsIfNeeded()`,
+ // either in the call directly above, or while materializing Phis. For all loads that did
+ // not have a substitute recorded, record it now; this can also be a type conversion.
+ HInstruction* substitute = FindSubstitute(load);
+ if (substitute == load) {
+ AddRemovedLoad(load, heap_value);
+ } else {
+ DCHECK(substitute->IsTypeConversion());
+ }
}
}
}
@@ -2495,7 +2760,8 @@ void LSEVisitor::UpdateValueRecordForStoreElimination(/*inout*/ValueRecord* valu
if (value_record->value.NeedsNonLoopPhi()) {
// Treat all Phi placeholders as requiring loop Phis at this point.
// We do not want MaterializeLoopPhis() to call MaterializeNonLoopPhis().
- value_record->value = Value::ForLoopPhiPlaceholder(value_record->value.GetPhiPlaceholder());
+ value_record->value =
+ Value::ForPlainLoopPhiPlaceholder(value_record->value.GetPhiPlaceholder());
}
}
diff --git a/compiler/optimizing/load_store_elimination_test.cc b/compiler/optimizing/load_store_elimination_test.cc
index 347a050a3b..bb73688b89 100644
--- a/compiler/optimizing/load_store_elimination_test.cc
+++ b/compiler/optimizing/load_store_elimination_test.cc
@@ -1525,18 +1525,19 @@ TEST_P(TwoTypesConversionsTestGroup, StoreLoopLoadStoreLoad) {
EXPECT_INS_REMOVED(read1);
EXPECT_INS_REMOVED(read2);
- if (load_type1 != DataType::Type::kInt32 && load_type2 != load_type1) {
- GTEST_SKIP() << "FIXME: Missing type conversions. Bug: 341476044";
- }
- // Note: Sometimes we create two type conversions when one is enough (Int32->Int16->Int8).
- // We currently rely on the instruction simplifier to remove the intermediate conversion.
+ // Note: If the `load_type2` is not larger than the `load_type1`, we avoid
+ // the intermediate conversion and use `param` directly for the second load.
+ DataType::Type read2_input_type = DataType::Size(load_type2) <= DataType::Size(load_type1)
+ ? DataType::Type::kInt32
+ : load_type1;
HInstruction* current = ret->InputAt(0);
- if (!DataType::IsTypeConversionImplicit(load_type1, load_type2)) {
+ if (!DataType::IsTypeConversionImplicit(read2_input_type, load_type2)) {
ASSERT_TRUE(current->IsTypeConversion()) << current->DebugName();
ASSERT_EQ(load_type2, current->GetType());
current = current->InputAt(0);
}
- if (!DataType::IsTypeConversionImplicit(DataType::Type::kInt32, load_type1)) {
+ if (!DataType::IsTypeConversionImplicit(DataType::Type::kInt32, read2_input_type)) {
+ ASSERT_EQ(read2_input_type, load_type1);
ASSERT_TRUE(current->IsTypeConversion()) << current->DebugName();
ASSERT_EQ(load_type1, current->GetType()) << load_type2;
current = current->InputAt(0);
@@ -1589,7 +1590,6 @@ TEST_P(TwoTypesConversionsTestGroup, MergingConvertedValueStore) {
EXPECT_INS_REMOVED(phi_write) << "\n" << param_type << "/" << load_type;
ASSERT_EQ(param, ret_input) << ret_input->DebugName();
} else {
- GTEST_SKIP() << "FIXME: Missing type conversions. Bug: 341476044";
EXPECT_INS_RETAINED(phi_write) << "\n" << param_type << "/" << load_type;
ASSERT_TRUE(ret_input->IsPhi()) << ret_input->DebugName();
HInstruction* pre_header_input = ret_input->InputAt(0);
@@ -1654,22 +1654,25 @@ TEST_P(TwoTypesConversionsTestGroup, MergingTwiceConvertedValueStore) {
EXPECT_INS_REMOVED(phi_write) << "\n" << load_type1 << "/" << load_type2;
ASSERT_EQ(param, ret_input) << ret_input->DebugName();
} else {
- GTEST_SKIP() << "FIXME: Missing type conversions. Bug: 341476044";
EXPECT_INS_RETAINED(phi_write) << "\n" << load_type1 << "/" << load_type2;
ASSERT_TRUE(ret_input->IsPhi()) << ret_input->DebugName();
HInstruction* pre_header_input = ret_input->InputAt(0);
HInstruction* loop_body_input = ret_input->InputAt(1);
ASSERT_EQ(param, pre_header_input) << pre_header_input->DebugName();
ASSERT_TRUE(loop_body_input->IsTypeConversion());
- // Note: Sometimes we create two type conversions when one is enough (Int32->Int16->Int8).
- // We currently rely on the instruction simplifier to remove the intermediate conversion.
HInstruction* current = loop_body_input;
- if (!DataType::IsTypeConversionImplicit(load_type1, load_type2)) {
+ // Note: If the `load_type2` is not larger than the `load_type1`, we avoid
+ // the intermediate conversion and use Phi directly for the second load.
+ DataType::Type read2_input_type = DataType::Size(load_type2) <= DataType::Size(load_type1)
+ ? DataType::Type::kInt32
+ : load_type1;
+ if (!DataType::IsTypeConversionImplicit(read2_input_type, load_type2)) {
ASSERT_TRUE(current->IsTypeConversion()) << current->DebugName();
ASSERT_EQ(load_type2, current->GetType());
current = current->InputAt(0);
}
- if (!DataType::IsTypeConversionImplicit(DataType::Type::kInt32, load_type1)) {
+ if (!DataType::IsTypeConversionImplicit(DataType::Type::kInt32, read2_input_type)) {
+ ASSERT_EQ(read2_input_type, load_type1);
ASSERT_TRUE(current->IsTypeConversion()) << current->DebugName();
ASSERT_EQ(load_type1, current->GetType()) << load_type2;
current = current->InputAt(0);
@@ -1678,6 +1681,103 @@ TEST_P(TwoTypesConversionsTestGroup, MergingTwiceConvertedValueStore) {
}
}
+TEST_P(TwoTypesConversionsTestGroup, MergingConvertedValueStorePhiDeduplication) {
+ auto [load_type1, load_type2] = GetParam();
+ DataType::Type field_type1 = FieldTypeForLoadType(load_type1);
+ DataType::Type field_type2 = FieldTypeForLoadType(load_type2);
+ DataType::Type phi_field_type = DataType::Type::kInt32; // "phi field" can hold the full value.
+ CHECK(DataType::IsTypeConversionImplicit(load_type1, phi_field_type));
+ CHECK(DataType::IsTypeConversionImplicit(load_type2, phi_field_type));
+ DataType::Type param_type = DataType::Type::kInt32;
+
+ HBasicBlock* return_block = InitEntryMainExitGraph();
+ auto [pre_header, loop_header, loop_body] = CreateForLoopWithInstructions(return_block);
+
+ HInstruction* param = MakeParam(param_type);
+ HInstruction* object = MakeParam(DataType::Type::kReference);
+
+ // Initialize the two "phi fields".
+ HInstruction* pre_header_write1 =
+ MakeIFieldSet(pre_header, object, param, phi_field_type, MemberOffset(40));
+ HInstruction* pre_header_write2 =
+ MakeIFieldSet(pre_header, object, param, phi_field_type, MemberOffset(60));
+
+ // In the body, we read the "phi fields", store and load the values to different fields
+ // to force type conversion, and store back to the "phi fields".
+ HInstanceFieldGet* phi_read1 = MakeIFieldGet(loop_body, object, phi_field_type, MemberOffset(40));
+ HInstanceFieldGet* phi_read2 = MakeIFieldGet(loop_body, object, phi_field_type, MemberOffset(60));
+ HInstruction* conversion_write1 =
+ MakeIFieldSet(loop_body, object, phi_read1, field_type1, MemberOffset(32));
+ HInstruction* conversion_write2 =
+ MakeIFieldSet(loop_body, object, phi_read2, field_type2, MemberOffset(52));
+ HInstanceFieldGet* conversion_read1 =
+ MakeIFieldGet(loop_body, object, field_type1, MemberOffset(32));
+ conversion_read1->SetType(load_type1);
+ HInstanceFieldGet* conversion_read2 =
+ MakeIFieldGet(loop_body, object, field_type2, MemberOffset(52));
+ conversion_read2->SetType(load_type2);
+ HInstruction* phi_write1 =
+ MakeIFieldSet(loop_body, object, conversion_read1, phi_field_type, MemberOffset(40));
+ HInstruction* phi_write2 =
+ MakeIFieldSet(loop_body, object, conversion_read2, phi_field_type, MemberOffset(60));
+
+ HInstanceFieldGet* final_read1 =
+ MakeIFieldGet(return_block, object, phi_field_type, MemberOffset(40));
+ HInstanceFieldGet* final_read2 =
+ MakeIFieldGet(return_block, object, phi_field_type, MemberOffset(60));
+ HAdd* add = MakeBinOp<HAdd>(return_block, DataType::Type::kInt32, final_read1, final_read2);
+ HInstruction* ret = MakeReturn(return_block, add);
+
+ PerformLSE();
+
+ EXPECT_INS_RETAINED(pre_header_write1);
+ EXPECT_INS_RETAINED(pre_header_write2);
+ EXPECT_INS_RETAINED(conversion_write1);
+ EXPECT_INS_RETAINED(conversion_write2);
+ EXPECT_INS_REMOVED(phi_read1);
+ EXPECT_INS_REMOVED(phi_read2);
+ EXPECT_INS_REMOVED(conversion_read1);
+ EXPECT_INS_REMOVED(conversion_read2);
+ EXPECT_INS_REMOVED(final_read1);
+ EXPECT_INS_REMOVED(final_read2);
+
+ HInstruction* ret_input = ret->InputAt(0);
+ ASSERT_EQ(add, ret_input) << ret_input->DebugName();
+ HInstruction* add_lhs = add->InputAt(0);
+ HInstruction* add_rhs = add->InputAt(1);
+ if (load_type1 == load_type2) {
+ ASSERT_EQ(add_lhs, add_rhs);
+ } else {
+ ASSERT_NE(add_lhs, add_rhs);
+ }
+ if (DataType::IsTypeConversionImplicit(param_type, load_type1)) {
+ EXPECT_INS_REMOVED(phi_write1) << "\n" << load_type1 << "/" << load_type2;
+ ASSERT_EQ(param, add_lhs) << ret_input->DebugName();
+ } else {
+ EXPECT_INS_RETAINED(phi_write1) << "\n" << load_type1 << "/" << load_type2;
+ ASSERT_TRUE(add_lhs->IsPhi()) << add_lhs->DebugName();
+ HInstruction* pre_header_input = add_lhs->InputAt(0);
+ HInstruction* loop_body_input = add_lhs->InputAt(1);
+ ASSERT_EQ(param, pre_header_input) << pre_header_input->DebugName();
+ ASSERT_TRUE(loop_body_input->IsTypeConversion());
+ ASSERT_EQ(load_type1, loop_body_input->GetType());
+ ASSERT_EQ(add_lhs, loop_body_input->InputAt(0));
+ }
+ if (DataType::IsTypeConversionImplicit(param_type, load_type2)) {
+ EXPECT_INS_REMOVED(phi_write2) << "\n" << load_type1 << "/" << load_type2;
+ ASSERT_EQ(param, add_rhs) << ret_input->DebugName();
+ } else {
+ EXPECT_INS_RETAINED(phi_write2) << "\n" << load_type1 << "/" << load_type2;
+ ASSERT_TRUE(add_rhs->IsPhi()) << add_rhs->DebugName();
+ HInstruction* pre_header_input = add_rhs->InputAt(0);
+ HInstruction* loop_body_input = add_rhs->InputAt(1);
+ ASSERT_EQ(param, pre_header_input) << pre_header_input->DebugName();
+ ASSERT_TRUE(loop_body_input->IsTypeConversion());
+ ASSERT_EQ(load_type2, loop_body_input->GetType());
+ ASSERT_EQ(add_rhs, loop_body_input->InputAt(0));
+ }
+}
+
auto Int32AndSmallerTypesGenerator() {
return testing::Values(DataType::Type::kInt32,
DataType::Type::kInt16,
diff --git a/test/530-checker-lse/src/Main.java b/test/530-checker-lse/src/Main.java
index 2516129c6c..2a6a244679 100644
--- a/test/530-checker-lse/src/Main.java
+++ b/test/530-checker-lse/src/Main.java
@@ -4507,5 +4507,7 @@ public class Main {
assertLongEquals(testOverlapLoop(50), 7778742049l);
assertIntEquals($noinline$testPartialEscape1(new TestClass(), true), 1);
assertIntEquals($noinline$testPartialEscape1(new TestClass(), false), 0);
+
+ TypeConversions.main();
}
}
diff --git a/test/530-checker-lse/src/TypeConversions.java b/test/530-checker-lse/src/TypeConversions.java
new file mode 100644
index 0000000000..8901c7e282
--- /dev/null
+++ b/test/530-checker-lse/src/TypeConversions.java
@@ -0,0 +1,276 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TypeConversions {
+ static byte static_byte;
+ static char static_char;
+ static int static_int;
+ static int unrelated_static_int;
+
+ public static void assertIntEquals(int expected, int result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiStoreLoadConversionInt8(int) load_store_elimination (before)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ // The type conversion has already been eliminated.
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<Value>>] field_name:TypeConversions.static_byte
+ /// CHECK-DAG: StaticFieldSet field_name:TypeConversions.unrelated_static_int loop:B{{\d+}}
+ /// CHECK-DAG: <<GetB:b\d+>> StaticFieldGet field_name:TypeConversions.static_byte
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetB>>] field_name:TypeConversions.static_int
+ /// CHECK-DAG: <<GetI:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32
+ /// CHECK-DAG: Return [<<GetI>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiStoreLoadConversionInt8(int) load_store_elimination (after)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: <<Conv:b\d+>> TypeConversion [<<Value>>]
+ /// CHECK-DAG: Return [<<Conv>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiStoreLoadConversionInt8(int) load_store_elimination (after)
+ /// CHECK-NOT: StaticFieldGet
+ public static int $noinline$loopPhiStoreLoadConversionInt8(int value) {
+ static_byte = (byte) value;
+ // Irrelevant code but needed to make LSE use loop Phi placeholders.
+ for (int q = 1; q < 12; q++) {
+ unrelated_static_int = 24;
+ }
+ static_int = static_byte;
+ return static_int;
+ }
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiStoreLoadConversionUint8(int) load_store_elimination (before)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ // The type conversion has already been eliminated.
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<Value>>] field_name:TypeConversions.static_byte
+ /// CHECK-DAG: StaticFieldSet field_name:TypeConversions.unrelated_static_int loop:B{{\d+}}
+ // The `& 0xff` has already been merged with the load.
+ /// CHECK-DAG: <<GetA:a\d+>> StaticFieldGet field_name:TypeConversions.static_byte
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetA>>] field_name:TypeConversions.static_int
+ /// CHECK-DAG: <<GetI:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32
+ /// CHECK-DAG: Return [<<GetI>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiStoreLoadConversionUint8(int) load_store_elimination (after)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: <<Conv:a\d+>> TypeConversion [<<Value>>]
+ /// CHECK-DAG: Return [<<Conv>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiStoreLoadConversionUint8(int) load_store_elimination (after)
+ /// CHECK-NOT: StaticFieldGet
+ public static int $noinline$loopPhiStoreLoadConversionUint8(int value) {
+ static_byte = (byte) value;
+ // Irrelevant code but needed to make LSE use loop Phi placeholders.
+ for (int q = 1; q < 12; q++) {
+ unrelated_static_int = 24;
+ }
+ static_int = static_byte & 0xff;
+ return static_int;
+ }
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiTwoStoreLoadConversions(int) load_store_elimination (before)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ // The type conversion has already been eliminated.
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<Value>>] field_name:TypeConversions.static_byte
+ /// CHECK-DAG: StaticFieldSet field_name:TypeConversions.unrelated_static_int loop:B{{\d+}}
+ /// CHECK-DAG: <<GetB:b\d+>> StaticFieldGet field_name:TypeConversions.static_byte
+ // The type conversion has already been eliminated.
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetB>>] field_name:TypeConversions.static_int
+ /// CHECK-DAG: <<GetI1:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetI1>>] field_name:TypeConversions.static_char
+ /// CHECK-DAG: <<GetC:c\d+>> StaticFieldGet field_name:TypeConversions.static_char field_type:Uint16
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetC>>] field_name:TypeConversions.static_int
+ /// CHECK-DAG: <<GetI2:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32
+ /// CHECK-DAG: Return [<<GetI2>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiTwoStoreLoadConversions(int) load_store_elimination (after)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: <<ConvB:b\d+>> TypeConversion [<<Value>>]
+ /// CHECK-DAG: <<ConvC:c\d+>> TypeConversion [<<ConvB>>]
+ /// CHECK-DAG: Return [<<ConvC>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$loopPhiTwoStoreLoadConversions(int) load_store_elimination (after)
+ /// CHECK-NOT: StaticFieldGet
+ public static int $noinline$loopPhiTwoStoreLoadConversions(int value) {
+ static_byte = (byte) value;
+ // Irrelevant code but needed to make LSE use loop Phi placeholders.
+ for (int q = 1; q < 12; q++) {
+ unrelated_static_int = 24;
+ }
+ // Note: We need to go through `static_int` so that the instruction
+ // simplifier eliminates the type conversion to `char`.
+ // TODO: Improve the instruction simplifier to eliminate the conversion
+ // for `static_char = (char) static_byte`.
+ static_int = static_byte;
+ static_char = (char) static_int;
+ static_int = static_char;
+ return static_int;
+ }
+
+ /// CHECK-START: int TypeConversions.$noinline$conditionalStoreLoadConversionInt8InLoop(int, boolean) load_store_elimination (before)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<Value>>] field_name:TypeConversions.static_int
+ /// CHECK-DAG: <<GetI:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32 loop:<<Loop:B\d+>>
+ // The type conversion has already been eliminated.
+ /// CHECK-DAG: StaticFieldSet field_name:TypeConversions.static_byte loop:<<Loop>>
+ /// CHECK-DAG: <<GetB:b\d+>> StaticFieldGet field_name:TypeConversions.static_byte loop:<<Loop>>
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetB>>] field_name:TypeConversions.static_int loop:<<Loop>>
+ /// CHECK-DAG: <<GetI2:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32 loop:none
+ /// CHECK-DAG: Return [<<GetI2>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$conditionalStoreLoadConversionInt8InLoop(int, boolean) load_store_elimination (after)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Value>>,<<Phi2:i\d+>>] loop:<<Loop:B\d+>>
+ /// CHECK-DAG: <<Conv:b\d+>> TypeConversion [<<Phi1>>] loop:<<Loop>>
+ /// CHECK-DAG: <<Phi2>> Phi [<<Phi1>>,<<Conv>>] loop:<<Loop>>
+ /// CHECK-DAG: Return [<<Phi1>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$conditionalStoreLoadConversionInt8InLoop(int, boolean) load_store_elimination (after)
+ /// CHECK-NOT: StaticFieldGet
+ public static int $noinline$conditionalStoreLoadConversionInt8InLoop(int value, boolean cond) {
+ static_int = value;
+ for (int q = 1; q < 12; q++) {
+ if (cond) {
+ static_byte = (byte) static_int;
+ static_int = static_byte;
+ }
+ }
+ return static_int;
+ }
+
+ /// CHECK-START: int TypeConversions.$noinline$conditionalStoreLoadConversionUint8InLoop(int, boolean) load_store_elimination (before)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<Value>>] field_name:TypeConversions.static_int
+ /// CHECK-DAG: <<GetI1:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32 loop:<<Loop:B\d+>>
+ // The type conversion has already been eliminated.
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetI1>>] field_name:TypeConversions.static_byte loop:<<Loop>>
+ /// CHECK-DAG: <<GetA:a\d+>> StaticFieldGet field_name:TypeConversions.static_byte loop:<<Loop>>
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetA>>] field_name:TypeConversions.static_int loop:<<Loop>>
+ /// CHECK-DAG: <<GetI2:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32 loop:none
+ /// CHECK-DAG: Return [<<GetI2>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$conditionalStoreLoadConversionUint8InLoop(int, boolean) load_store_elimination (after)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Value>>,<<Phi2:i\d+>>] loop:<<Loop:B\d+>>
+ /// CHECK-DAG: <<Conv:a\d+>> TypeConversion [<<Phi1>>] loop:<<Loop>>
+ /// CHECK-DAG: <<Phi2>> Phi [<<Phi1>>,<<Conv>>] loop:<<Loop>>
+ /// CHECK-DAG: Return [<<Phi1>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$conditionalStoreLoadConversionUint8InLoop(int, boolean) load_store_elimination (after)
+ /// CHECK-NOT: StaticFieldGet
+ public static int $noinline$conditionalStoreLoadConversionUint8InLoop(int value, boolean cond) {
+ static_int = value;
+ for (int q = 1; q < 12; q++) {
+ if (cond) {
+ static_byte = (byte) static_int;
+ static_int = static_byte & 0xff;
+ }
+ }
+ return static_int;
+ }
+
+ /// CHECK-START: int TypeConversions.$noinline$twoConditionalStoreLoadConversionsInLoop(int, boolean) load_store_elimination (before)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<Value>>] field_name:TypeConversions.static_int
+ /// CHECK-DAG: <<GetI1:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32 loop:<<Loop:B\d+>>
+ // The type conversion has already been eliminated.
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetI1>>] field_name:TypeConversions.static_byte loop:<<Loop>>
+ /// CHECK-DAG: <<GetB:b\d+>> StaticFieldGet field_name:TypeConversions.static_byte loop:<<Loop>>
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetB>>] field_name:TypeConversions.static_int loop:<<Loop>>
+ /// CHECK-DAG: <<GetI2:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32 loop:<<Loop>>
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetI2>>] field_name:TypeConversions.static_char loop:<<Loop>>
+ /// CHECK-DAG: <<GetC:c\d+>> StaticFieldGet field_name:TypeConversions.static_char loop:<<Loop>>
+ /// CHECK-DAG: StaticFieldSet [{{l\d+}},<<GetC>>] field_name:TypeConversions.static_int loop:<<Loop>>
+ /// CHECK-DAG: <<GetI3:i\d+>> StaticFieldGet field_name:TypeConversions.static_int field_type:Int32 loop:none
+ /// CHECK-DAG: Return [<<GetI3>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$twoConditionalStoreLoadConversionsInLoop(int, boolean) load_store_elimination (after)
+ /// CHECK-DAG: <<Value:i\d+>> ParameterValue
+ /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Value>>,<<Phi2:i\d+>>] loop:<<Loop:B\d+>>
+ /// CHECK-DAG: <<Conv1:b\d+>> TypeConversion [<<Phi1>>] loop:<<Loop>>
+ /// CHECK-DAG: <<Conv2:c\d+>> TypeConversion [<<Conv1>>] loop:<<Loop>>
+ /// CHECK-DAG: <<Phi2>> Phi [<<Phi1>>,<<Conv2>>] loop:<<Loop>>
+ /// CHECK-DAG: Return [<<Phi1>>]
+
+ /// CHECK-START: int TypeConversions.$noinline$twoConditionalStoreLoadConversionsInLoop(int, boolean) load_store_elimination (after)
+ /// CHECK-NOT: StaticFieldGet
+ public static int $noinline$twoConditionalStoreLoadConversionsInLoop(int value, boolean cond) {
+ static_int = value;
+ for (int q = 1; q < 12; q++) {
+ if (cond) {
+ static_byte = (byte) static_int;
+ // Note: We need to go through `static_int` so that the instruction
+ // simplifier eliminates the type conversion to `char`.
+ // TODO: Improve the instruction simplifier to eliminate the conversion
+ // for `static_char = (char) static_byte`.
+ static_int = static_byte;
+ static_char = (char) static_int;
+ static_int = static_char;
+ }
+ }
+ return static_int;
+ }
+
+ public static void main() {
+ assertIntEquals(42, $noinline$loopPhiStoreLoadConversionInt8(42));
+ assertIntEquals(-42, $noinline$loopPhiStoreLoadConversionInt8(-42));
+ assertIntEquals(-128, $noinline$loopPhiStoreLoadConversionInt8(128));
+ assertIntEquals(127, $noinline$loopPhiStoreLoadConversionInt8(-129));
+
+ assertIntEquals(42, $noinline$loopPhiStoreLoadConversionUint8(42));
+ assertIntEquals(214, $noinline$loopPhiStoreLoadConversionUint8(-42));
+ assertIntEquals(128, $noinline$loopPhiStoreLoadConversionUint8(128));
+ assertIntEquals(127, $noinline$loopPhiStoreLoadConversionUint8(-129));
+
+ assertIntEquals(42, $noinline$loopPhiTwoStoreLoadConversions(42));
+ assertIntEquals(65494, $noinline$loopPhiTwoStoreLoadConversions(-42));
+ assertIntEquals(65408, $noinline$loopPhiTwoStoreLoadConversions(128));
+ assertIntEquals(127, $noinline$loopPhiTwoStoreLoadConversions(-129));
+ assertIntEquals(0, $noinline$loopPhiTwoStoreLoadConversions(256));
+ assertIntEquals(65535, $noinline$loopPhiTwoStoreLoadConversions(-257));
+
+ assertIntEquals(42, $noinline$conditionalStoreLoadConversionInt8InLoop(42, false));
+ assertIntEquals(42, $noinline$conditionalStoreLoadConversionInt8InLoop(42, true));
+ assertIntEquals(-42, $noinline$conditionalStoreLoadConversionInt8InLoop(-42, false));
+ assertIntEquals(-42, $noinline$conditionalStoreLoadConversionInt8InLoop(-42, true));
+ assertIntEquals(128, $noinline$conditionalStoreLoadConversionInt8InLoop(128, false));
+ assertIntEquals(-128, $noinline$conditionalStoreLoadConversionInt8InLoop(128, true));
+ assertIntEquals(-129, $noinline$conditionalStoreLoadConversionInt8InLoop(-129, false));
+ assertIntEquals(127, $noinline$conditionalStoreLoadConversionInt8InLoop(-129, true));
+
+ assertIntEquals(42, $noinline$conditionalStoreLoadConversionUint8InLoop(42, false));
+ assertIntEquals(42, $noinline$conditionalStoreLoadConversionUint8InLoop(42, true));
+ assertIntEquals(-42, $noinline$conditionalStoreLoadConversionUint8InLoop(-42, false));
+ assertIntEquals(214, $noinline$conditionalStoreLoadConversionUint8InLoop(-42, true));
+ assertIntEquals(128, $noinline$conditionalStoreLoadConversionUint8InLoop(128, false));
+ assertIntEquals(128, $noinline$conditionalStoreLoadConversionUint8InLoop(128, true));
+ assertIntEquals(-129, $noinline$conditionalStoreLoadConversionUint8InLoop(-129, false));
+ assertIntEquals(127, $noinline$conditionalStoreLoadConversionUint8InLoop(-129, true));
+
+ assertIntEquals(42, $noinline$twoConditionalStoreLoadConversionsInLoop(42, false));
+ assertIntEquals(42, $noinline$twoConditionalStoreLoadConversionsInLoop(42, true));
+ assertIntEquals(-42, $noinline$twoConditionalStoreLoadConversionsInLoop(-42, false));
+ assertIntEquals(65494, $noinline$twoConditionalStoreLoadConversionsInLoop(-42, true));
+ assertIntEquals(128, $noinline$twoConditionalStoreLoadConversionsInLoop(128, false));
+ assertIntEquals(65408, $noinline$twoConditionalStoreLoadConversionsInLoop(128, true));
+ assertIntEquals(-129, $noinline$twoConditionalStoreLoadConversionsInLoop(-129, false));
+ assertIntEquals(127, $noinline$twoConditionalStoreLoadConversionsInLoop(-129, true));
+ assertIntEquals(256, $noinline$twoConditionalStoreLoadConversionsInLoop(256, false));
+ assertIntEquals(0, $noinline$twoConditionalStoreLoadConversionsInLoop(256, true));
+ assertIntEquals(-257, $noinline$twoConditionalStoreLoadConversionsInLoop(-257, false));
+ assertIntEquals(65535, $noinline$twoConditionalStoreLoadConversionsInLoop(-257, true));
+ }
+}