diff options
Diffstat (limited to 'compiler')
69 files changed, 2691 insertions, 1091 deletions
diff --git a/compiler/Android.mk b/compiler/Android.mk index f12f00743b..9a416e25a1 100644 --- a/compiler/Android.mk +++ b/compiler/Android.mk @@ -148,6 +148,7 @@ LIBART_COMPILER_SRC_FILES_x86_64 := \ LIBART_COMPILER_CFLAGS := LIBART_COMPILER_ENUM_OPERATOR_OUT_HEADER_FILES := \ + compiled_method.h \ dex/compiler_enums.h \ dex/dex_to_dex_compiler.h \ driver/compiler_driver.h \ @@ -299,28 +300,6 @@ $$(ENUM_OPERATOR_OUT_GEN): $$(GENERATED_SRC_DIR)/%_operator_out.cc : $(LOCAL_PAT endif endif - ifeq ($$(art_target_or_host),target) - ifeq ($$(art_ndebug_or_debug),debug) - $(TARGET_OUT_EXECUTABLES)/dex2oatd: $$(LOCAL_INSTALLED_MODULE) - else - $(TARGET_OUT_EXECUTABLES)/dex2oat: $$(LOCAL_INSTALLED_MODULE) - endif - else # host - ifeq ($$(art_ndebug_or_debug),debug) - ifeq ($$(art_static_or_shared),static) - $(HOST_OUT_EXECUTABLES)/dex2oatds: $$(LOCAL_INSTALLED_MODULE) - else - $(HOST_OUT_EXECUTABLES)/dex2oatd: $$(LOCAL_INSTALLED_MODULE) - endif - else - ifeq ($$(art_static_or_shared),static) - $(HOST_OUT_EXECUTABLES)/dex2oats: $$(LOCAL_INSTALLED_MODULE) - else - $(HOST_OUT_EXECUTABLES)/dex2oat: $$(LOCAL_INSTALLED_MODULE) - endif - endif - endif - # Clear locally defined variables. art_target_or_host := art_ndebug_or_debug := diff --git a/compiler/compiled_method.h b/compiler/compiled_method.h index 58876200ca..b5c99e8c95 100644 --- a/compiler/compiled_method.h +++ b/compiler/compiled_method.h @@ -18,6 +18,7 @@ #define ART_COMPILER_COMPILED_METHOD_H_ #include <memory> +#include <iosfwd> #include <string> #include <vector> @@ -158,21 +159,27 @@ class SrcMap FINAL : public std::vector<SrcMapElem, Allocator> { using DefaultSrcMap = SrcMap<std::allocator<SrcMapElem>>; - -enum LinkerPatchType { - kLinkerPatchMethod, - kLinkerPatchCall, - kLinkerPatchCallRelative, // NOTE: Actual patching is instruction_set-dependent. - kLinkerPatchType, - kLinkerPatchDexCacheArray, // NOTE: Actual patching is instruction_set-dependent. -}; - class LinkerPatch { public: + enum class Type { + kRecordPosition, // Just record patch position for patchoat. + kMethod, + kCall, + kCallRelative, // NOTE: Actual patching is instruction_set-dependent. + kType, + kString, + kStringRelative, // NOTE: Actual patching is instruction_set-dependent. + kDexCacheArray, // NOTE: Actual patching is instruction_set-dependent. + }; + + static LinkerPatch RecordPosition(size_t literal_offset) { + return LinkerPatch(literal_offset, Type::kRecordPosition, /* target_dex_file */ nullptr); + } + static LinkerPatch MethodPatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t target_method_idx) { - LinkerPatch patch(literal_offset, kLinkerPatchMethod, target_dex_file); + LinkerPatch patch(literal_offset, Type::kMethod, target_dex_file); patch.method_idx_ = target_method_idx; return patch; } @@ -180,7 +187,7 @@ class LinkerPatch { static LinkerPatch CodePatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t target_method_idx) { - LinkerPatch patch(literal_offset, kLinkerPatchCall, target_dex_file); + LinkerPatch patch(literal_offset, Type::kCall, target_dex_file); patch.method_idx_ = target_method_idx; return patch; } @@ -188,7 +195,7 @@ class LinkerPatch { static LinkerPatch RelativeCodePatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t target_method_idx) { - LinkerPatch patch(literal_offset, kLinkerPatchCallRelative, target_dex_file); + LinkerPatch patch(literal_offset, Type::kCallRelative, target_dex_file); patch.method_idx_ = target_method_idx; return patch; } @@ -196,17 +203,35 @@ class LinkerPatch { static LinkerPatch TypePatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t target_type_idx) { - LinkerPatch patch(literal_offset, kLinkerPatchType, target_dex_file); + LinkerPatch patch(literal_offset, Type::kType, target_dex_file); patch.type_idx_ = target_type_idx; return patch; } + static LinkerPatch StringPatch(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t target_string_idx) { + LinkerPatch patch(literal_offset, Type::kString, target_dex_file); + patch.string_idx_ = target_string_idx; + return patch; + } + + static LinkerPatch RelativeStringPatch(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t target_string_idx) { + LinkerPatch patch(literal_offset, Type::kStringRelative, target_dex_file); + patch.string_idx_ = target_string_idx; + patch.pc_insn_offset_ = pc_insn_offset; + return patch; + } + static LinkerPatch DexCacheArrayPatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t pc_insn_offset, size_t element_offset) { DCHECK(IsUint<32>(element_offset)); - LinkerPatch patch(literal_offset, kLinkerPatchDexCacheArray, target_dex_file); + LinkerPatch patch(literal_offset, Type::kDexCacheArray, target_dex_file); patch.pc_insn_offset_ = pc_insn_offset; patch.element_offset_ = element_offset; return patch; @@ -219,47 +244,65 @@ class LinkerPatch { return literal_offset_; } - LinkerPatchType Type() const { + Type GetType() const { return patch_type_; } bool IsPcRelative() const { - return Type() == kLinkerPatchCallRelative || Type() == kLinkerPatchDexCacheArray; + switch (GetType()) { + case Type::kCallRelative: + case Type::kStringRelative: + case Type::kDexCacheArray: + return true; + default: + return false; + } } MethodReference TargetMethod() const { - DCHECK(patch_type_ == kLinkerPatchMethod || - patch_type_ == kLinkerPatchCall || patch_type_ == kLinkerPatchCallRelative); + DCHECK(patch_type_ == Type::kMethod || + patch_type_ == Type::kCall || + patch_type_ == Type::kCallRelative); return MethodReference(target_dex_file_, method_idx_); } const DexFile* TargetTypeDexFile() const { - DCHECK(patch_type_ == kLinkerPatchType); + DCHECK(patch_type_ == Type::kType); return target_dex_file_; } uint32_t TargetTypeIndex() const { - DCHECK(patch_type_ == kLinkerPatchType); + DCHECK(patch_type_ == Type::kType); return type_idx_; } + const DexFile* TargetStringDexFile() const { + DCHECK(patch_type_ == Type::kString || patch_type_ == Type::kStringRelative); + return target_dex_file_; + } + + uint32_t TargetStringIndex() const { + DCHECK(patch_type_ == Type::kString || patch_type_ == Type::kStringRelative); + return string_idx_; + } + const DexFile* TargetDexCacheDexFile() const { - DCHECK(patch_type_ == kLinkerPatchDexCacheArray); + DCHECK(patch_type_ == Type::kDexCacheArray); return target_dex_file_; } size_t TargetDexCacheElementOffset() const { - DCHECK(patch_type_ == kLinkerPatchDexCacheArray); + DCHECK(patch_type_ == Type::kDexCacheArray); return element_offset_; } uint32_t PcInsnOffset() const { - DCHECK(patch_type_ == kLinkerPatchDexCacheArray); + DCHECK(patch_type_ == Type::kStringRelative || patch_type_ == Type::kDexCacheArray); return pc_insn_offset_; } private: - LinkerPatch(size_t literal_offset, LinkerPatchType patch_type, const DexFile* target_dex_file) + LinkerPatch(size_t literal_offset, Type patch_type, const DexFile* target_dex_file) : target_dex_file_(target_dex_file), literal_offset_(literal_offset), patch_type_(patch_type) { @@ -272,14 +315,16 @@ class LinkerPatch { const DexFile* target_dex_file_; uint32_t literal_offset_ : 24; // Method code size up to 16MiB. - LinkerPatchType patch_type_ : 8; + Type patch_type_ : 8; union { uint32_t cmp1_; // Used for relational operators. uint32_t method_idx_; // Method index for Call/Method patches. uint32_t type_idx_; // Type index for Type patches. + uint32_t string_idx_; // String index for String patches. uint32_t element_offset_; // Element offset in the dex cache arrays. static_assert(sizeof(method_idx_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(type_idx_) == sizeof(cmp1_), "needed by relational operators"); + static_assert(sizeof(string_idx_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(element_offset_) == sizeof(cmp1_), "needed by relational operators"); }; union { @@ -295,6 +340,7 @@ class LinkerPatch { friend bool operator==(const LinkerPatch& lhs, const LinkerPatch& rhs); friend bool operator<(const LinkerPatch& lhs, const LinkerPatch& rhs); }; +std::ostream& operator<<(std::ostream& os, const LinkerPatch::Type& type); inline bool operator==(const LinkerPatch& lhs, const LinkerPatch& rhs) { return lhs.literal_offset_ == rhs.literal_offset_ && diff --git a/compiler/compiler.h b/compiler/compiler.h index 97c60de8c0..487a27fec0 100644 --- a/compiler/compiler.h +++ b/compiler/compiler.h @@ -27,8 +27,6 @@ namespace jit { } class ArtMethod; -class Backend; -struct CompilationUnit; class CompilerDriver; class CompiledMethod; class OatWriter; @@ -46,8 +44,7 @@ class Compiler { virtual void UnInit() const = 0; - virtual bool CanCompileMethod(uint32_t method_idx, const DexFile& dex_file, CompilationUnit* cu) - const = 0; + virtual bool CanCompileMethod(uint32_t method_idx, const DexFile& dex_file) const = 0; virtual CompiledMethod* Compile(const DexFile::CodeItem* code_item, uint32_t access_flags, @@ -77,8 +74,6 @@ class Compiler { return maximum_compilation_time_before_warning_; } - virtual void InitCompilationUnit(CompilationUnit& cu) const = 0; - virtual ~Compiler() {} /* diff --git a/compiler/dex/dex_to_dex_compiler.cc b/compiler/dex/dex_to_dex_compiler.cc index 4836041ce0..efddeba6a9 100644 --- a/compiler/dex/dex_to_dex_compiler.cc +++ b/compiler/dex/dex_to_dex_compiler.cc @@ -327,10 +327,16 @@ CompiledMethod* ArtCompileDEX( ScopedObjectAccess soa(Thread::Current()); StackHandleScope<1> hs(soa.Self()); ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); - art::DexCompilationUnit unit(nullptr, class_loader, class_linker, - dex_file, code_item, class_def_idx, method_idx, access_flags, - driver->GetVerifiedMethod(&dex_file, method_idx), - hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file))); + art::DexCompilationUnit unit( + class_loader, + class_linker, + dex_file, + code_item, + class_def_idx, + method_idx, + access_flags, + driver->GetVerifiedMethod(&dex_file, method_idx), + hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file))); art::optimizer::DexCompiler dex_compiler(*driver, unit, dex_to_dex_compilation_level); dex_compiler.Compile(); if (dex_compiler.GetQuickenedInfo().empty()) { diff --git a/compiler/dex/verification_results.cc b/compiler/dex/verification_results.cc index 7c9ce1ee60..1491a183f1 100644 --- a/compiler/dex/verification_results.cc +++ b/compiler/dex/verification_results.cc @@ -64,7 +64,6 @@ void VerificationResults::ProcessVerifiedMethod(verifier::MethodVerifier* method DCHECK_EQ(it->second->GetDevirtMap().size(), verified_method->GetDevirtMap().size()); DCHECK_EQ(it->second->GetSafeCastSet().size(), verified_method->GetSafeCastSet().size()); } - DCHECK_EQ(it->second->GetDexGcMap().size(), verified_method->GetDexGcMap().size()); // Delete the new verified method since there was already an existing one registered. It // is unsafe to replace the existing one since the JIT may be using it to generate a // native GC map. diff --git a/compiler/dex/verified_method.cc b/compiler/dex/verified_method.cc index 9ae21648bf..5c0253c29e 100644 --- a/compiler/dex/verified_method.cc +++ b/compiler/dex/verified_method.cc @@ -30,7 +30,6 @@ #include "mirror/dex_cache-inl.h" #include "mirror/object-inl.h" #include "utils.h" -#include "verifier/dex_gc_map.h" #include "verifier/method_verifier-inl.h" #include "verifier/reg_type-inl.h" #include "verifier/register_line-inl.h" @@ -49,14 +48,6 @@ const VerifiedMethod* VerifiedMethod::Create(verifier::MethodVerifier* method_ve method_verifier->HasInstructionThatWillThrow())); if (compile) { - /* Generate a register map. */ - if (!verified_method->GenerateGcMap(method_verifier)) { - return nullptr; // Not a real failure, but a failure to encode. - } - if (kIsDebugBuild) { - VerifyGcMap(method_verifier, verified_method->dex_gc_map_); - } - // TODO: move this out when DEX-to-DEX supports devirtualization. if (method_verifier->HasVirtualOrInterfaceInvokes()) { verified_method->GenerateDevirtMap(method_verifier); @@ -90,120 +81,6 @@ bool VerifiedMethod::IsSafeCast(uint32_t pc) const { return std::binary_search(safe_cast_set_.begin(), safe_cast_set_.end(), pc); } -bool VerifiedMethod::GenerateGcMap(verifier::MethodVerifier* method_verifier) { - DCHECK(dex_gc_map_.empty()); - size_t num_entries, ref_bitmap_bits, pc_bits; - ComputeGcMapSizes(method_verifier, &num_entries, &ref_bitmap_bits, &pc_bits); - const size_t ref_bitmap_bytes = RoundUp(ref_bitmap_bits, kBitsPerByte) / kBitsPerByte; - static constexpr size_t kFormatBits = 3; - // We have 16 - kFormatBits available for the ref_bitmap_bytes. - if ((ref_bitmap_bytes >> (16u - kFormatBits)) != 0) { - LOG(WARNING) << "Cannot encode GC map for method with " << ref_bitmap_bits << " registers: " - << PrettyMethod(method_verifier->GetMethodReference().dex_method_index, - *method_verifier->GetMethodReference().dex_file); - return false; - } - // There are 2 bytes to encode the number of entries. - if (num_entries > std::numeric_limits<uint16_t>::max()) { - LOG(WARNING) << "Cannot encode GC map for method with " << num_entries << " entries: " - << PrettyMethod(method_verifier->GetMethodReference().dex_method_index, - *method_verifier->GetMethodReference().dex_file); - return false; - } - size_t pc_bytes; - verifier::RegisterMapFormat format; - if (pc_bits <= kBitsPerByte) { - format = verifier::kRegMapFormatCompact8; - pc_bytes = 1; - } else if (pc_bits <= kBitsPerByte * 2) { - format = verifier::kRegMapFormatCompact16; - pc_bytes = 2; - } else { - LOG(WARNING) << "Cannot encode GC map for method with " - << (1 << pc_bits) << " instructions (number is rounded up to nearest power of 2): " - << PrettyMethod(method_verifier->GetMethodReference().dex_method_index, - *method_verifier->GetMethodReference().dex_file); - return false; - } - size_t table_size = ((pc_bytes + ref_bitmap_bytes) * num_entries) + 4; - dex_gc_map_.reserve(table_size); - // Write table header. - dex_gc_map_.push_back(format | ((ref_bitmap_bytes & ~0xFF) >> (kBitsPerByte - kFormatBits))); - dex_gc_map_.push_back(ref_bitmap_bytes & 0xFF); - dex_gc_map_.push_back(num_entries & 0xFF); - dex_gc_map_.push_back((num_entries >> 8) & 0xFF); - // Write table data. - const DexFile::CodeItem* code_item = method_verifier->CodeItem(); - for (size_t i = 0; i < code_item->insns_size_in_code_units_; i++) { - if (method_verifier->GetInstructionFlags(i).IsCompileTimeInfoPoint()) { - dex_gc_map_.push_back(i & 0xFF); - if (pc_bytes == 2) { - dex_gc_map_.push_back((i >> 8) & 0xFF); - } - verifier::RegisterLine* line = method_verifier->GetRegLine(i); - line->WriteReferenceBitMap(method_verifier, &dex_gc_map_, ref_bitmap_bytes); - } - } - DCHECK_EQ(dex_gc_map_.size(), table_size); - return true; -} - -void VerifiedMethod::VerifyGcMap(verifier::MethodVerifier* method_verifier, - const std::vector<uint8_t>& data) { - // Check that for every GC point there is a map entry, there aren't entries for non-GC points, - // that the table data is well formed and all references are marked (or not) in the bitmap. - verifier::DexPcToReferenceMap map(&data[0]); - CHECK_EQ(data.size(), map.RawSize()) << map.NumEntries() << " " << map.RegWidth(); - size_t map_index = 0; - const DexFile::CodeItem* code_item = method_verifier->CodeItem(); - for (size_t i = 0; i < code_item->insns_size_in_code_units_; i++) { - const uint8_t* reg_bitmap = map.FindBitMap(i, false); - if (method_verifier->GetInstructionFlags(i).IsCompileTimeInfoPoint()) { - DCHECK_LT(map_index, map.NumEntries()); - DCHECK_EQ(map.GetDexPc(map_index), i); - DCHECK_EQ(map.GetBitMap(map_index), reg_bitmap); - map_index++; - verifier::RegisterLine* line = method_verifier->GetRegLine(i); - for (size_t j = 0; j < code_item->registers_size_; j++) { - if (line->GetRegisterType(method_verifier, j).IsNonZeroReferenceTypes()) { - DCHECK_LT(j / kBitsPerByte, map.RegWidth()); - DCHECK_EQ((reg_bitmap[j / kBitsPerByte] >> (j % kBitsPerByte)) & 1, 1); - } else if ((j / kBitsPerByte) < map.RegWidth()) { - DCHECK_EQ((reg_bitmap[j / kBitsPerByte] >> (j % kBitsPerByte)) & 1, 0); - } else { - // If a register doesn't contain a reference then the bitmap may be shorter than the line. - } - } - } else { - DCHECK(i >= 65536 || reg_bitmap == nullptr); - } - } -} - -void VerifiedMethod::ComputeGcMapSizes(verifier::MethodVerifier* method_verifier, - size_t* gc_points, size_t* ref_bitmap_bits, - size_t* log2_max_gc_pc) { - size_t local_gc_points = 0; - size_t max_insn = 0; - size_t max_ref_reg = -1; - const DexFile::CodeItem* code_item = method_verifier->CodeItem(); - for (size_t i = 0; i < code_item->insns_size_in_code_units_; i++) { - if (method_verifier->GetInstructionFlags(i).IsCompileTimeInfoPoint()) { - local_gc_points++; - max_insn = i; - verifier::RegisterLine* line = method_verifier->GetRegLine(i); - max_ref_reg = line->GetMaxNonZeroReferenceReg(method_verifier, max_ref_reg); - } - } - *gc_points = local_gc_points; - *ref_bitmap_bits = max_ref_reg + 1; // If max register is 0 we need 1 bit to encode (ie +1). - size_t i = 0; - while ((1U << i) <= max_insn) { - i++; - } - *log2_max_gc_pc = i; -} - bool VerifiedMethod::GenerateDequickenMap(verifier::MethodVerifier* method_verifier) { if (method_verifier->HasFailures()) { return false; diff --git a/compiler/dex/verified_method.h b/compiler/dex/verified_method.h index 12d0219058..495acf07b0 100644 --- a/compiler/dex/verified_method.h +++ b/compiler/dex/verified_method.h @@ -47,10 +47,6 @@ class VerifiedMethod { SHARED_REQUIRES(Locks::mutator_lock_); ~VerifiedMethod() = default; - const std::vector<uint8_t>& GetDexGcMap() const { - return dex_gc_map_; - } - const DevirtualizationMap& GetDevirtMap() const { return devirt_map_; } @@ -114,7 +110,6 @@ class VerifiedMethod { void GenerateSafeCastSet(verifier::MethodVerifier* method_verifier) SHARED_REQUIRES(Locks::mutator_lock_); - std::vector<uint8_t> dex_gc_map_; DevirtualizationMap devirt_map_; // Dequicken map is required for compiling quickened byte codes. The quicken maps from // dex PC to dex method index or dex field index based on the instruction. diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 5c1c92b712..5bfd5f3189 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -2210,7 +2210,8 @@ void CompilerDriver::Verify(jobject class_loader, class VerifyClassVisitor : public CompilationVisitor { public: - explicit VerifyClassVisitor(const ParallelCompilationManager* manager) : manager_(manager) {} + VerifyClassVisitor(const ParallelCompilationManager* manager, LogSeverity log_level) + : manager_(manager), log_level_(log_level) {} virtual void Visit(size_t class_def_index) REQUIRES(!Locks::mutator_lock_) OVERRIDE { ATRACE_CALL(); @@ -2248,7 +2249,7 @@ class VerifyClassVisitor : public CompilationVisitor { &class_def, Runtime::Current()->GetCompilerCallbacks(), true /* allow soft failures */, - true /* log hard failures */, + log_level_, &error_msg) == verifier::MethodVerifier::kHardFailure) { LOG(ERROR) << "Verification failed on class " << PrettyDescriptor(descriptor) @@ -2257,7 +2258,7 @@ class VerifyClassVisitor : public CompilationVisitor { } } else if (!SkipClass(jclass_loader, dex_file, klass.Get())) { CHECK(klass->IsResolved()) << PrettyClass(klass.Get()); - class_linker->VerifyClass(soa.Self(), klass); + class_linker->VerifyClass(soa.Self(), klass, log_level_); if (klass->IsErroneous()) { // ClassLinker::VerifyClass throws, which isn't useful in the compiler. @@ -2280,6 +2281,7 @@ class VerifyClassVisitor : public CompilationVisitor { private: const ParallelCompilationManager* const manager_; + const LogSeverity log_level_; }; void CompilerDriver::VerifyDexFile(jobject class_loader, @@ -2292,7 +2294,10 @@ void CompilerDriver::VerifyDexFile(jobject class_loader, ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); ParallelCompilationManager context(class_linker, class_loader, this, &dex_file, dex_files, thread_pool); - VerifyClassVisitor visitor(&context); + LogSeverity log_level = GetCompilerOptions().AbortOnHardVerifierFailure() + ? LogSeverity::INTERNAL_FATAL + : LogSeverity::WARNING; + VerifyClassVisitor visitor(&context, log_level); context.ForAll(0, dex_file.NumClassDefs(), &visitor, thread_count); } diff --git a/compiler/driver/dex_compilation_unit.cc b/compiler/driver/dex_compilation_unit.cc index e458b98319..b0ee448ea6 100644 --- a/compiler/driver/dex_compilation_unit.cc +++ b/compiler/driver/dex_compilation_unit.cc @@ -22,8 +22,7 @@ namespace art { -DexCompilationUnit::DexCompilationUnit(CompilationUnit* cu, - jobject class_loader, +DexCompilationUnit::DexCompilationUnit(jobject class_loader, ClassLinker* class_linker, const DexFile& dex_file, const DexFile::CodeItem* code_item, @@ -32,8 +31,7 @@ DexCompilationUnit::DexCompilationUnit(CompilationUnit* cu, uint32_t access_flags, const VerifiedMethod* verified_method, Handle<mirror::DexCache> dex_cache) - : cu_(cu), - class_loader_(class_loader), + : class_loader_(class_loader), class_linker_(class_linker), dex_file_(&dex_file), code_item_(code_item), diff --git a/compiler/driver/dex_compilation_unit.h b/compiler/driver/dex_compilation_unit.h index 16872f41f3..854927d747 100644 --- a/compiler/driver/dex_compilation_unit.h +++ b/compiler/driver/dex_compilation_unit.h @@ -30,15 +30,11 @@ class ClassLoader; class DexCache; } // namespace mirror class ClassLinker; -struct CompilationUnit; class VerifiedMethod; class DexCompilationUnit : public DeletableArenaObject<kArenaAllocMisc> { public: - explicit DexCompilationUnit(CompilationUnit* cu); - - DexCompilationUnit(CompilationUnit* cu, - jobject class_loader, + DexCompilationUnit(jobject class_loader, ClassLinker* class_linker, const DexFile& dex_file, const DexFile::CodeItem* code_item, @@ -48,10 +44,6 @@ class DexCompilationUnit : public DeletableArenaObject<kArenaAllocMisc> { const VerifiedMethod* verified_method, Handle<mirror::DexCache> dex_cache); - CompilationUnit* GetCompilationUnit() const { - return cu_; - } - jobject GetClassLoader() const { return class_loader_; } @@ -121,8 +113,6 @@ class DexCompilationUnit : public DeletableArenaObject<kArenaAllocMisc> { } private: - CompilationUnit* const cu_; - const jobject class_loader_; ClassLinker* const class_linker_; diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc index 682b008219..d4dd978c5c 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.cc +++ b/compiler/linker/arm/relative_patcher_arm_base.cc @@ -112,7 +112,7 @@ uint32_t ArmBaseRelativePatcher::ReserveSpaceInternal(uint32_t offset, } } for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.Type() == kLinkerPatchCallRelative) { + if (patch.GetType() == LinkerPatch::Type::kCallRelative) { unprocessed_patches_.emplace_back(patch.TargetMethod(), quick_code_offset + patch.LiteralOffset()); } diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc index c090dffc55..582ecb3bbf 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.cc +++ b/compiler/linker/arm/relative_patcher_thumb2.cc @@ -56,10 +56,10 @@ void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code, SetInsn32(code, literal_offset, value); } -void Thumb2RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) { +void Thumb2RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) { uint32_t literal_offset = patch.LiteralOffset(); uint32_t pc_literal_offset = patch.PcInsnOffset(); uint32_t pc_base = patch_offset + (pc_literal_offset - literal_offset) + 4u /* PC adjustment */; diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h index 0d903c0b41..d85739c51f 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.h +++ b/compiler/linker/arm/relative_patcher_thumb2.h @@ -30,10 +30,10 @@ class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { uint32_t literal_offset, uint32_t patch_offset, uint32_t target_offset) OVERRIDE; - void PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) OVERRIDE; + void PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) OVERRIDE; private: static std::vector<uint8_t> CompileThunkCode(); diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc index a259cda986..a8078e3049 100644 --- a/compiler/linker/arm/relative_patcher_thumb2_test.cc +++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc @@ -30,6 +30,9 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { static const ArrayRef<const uint8_t> kCallCode; static const uint8_t kNopRawCode[]; static const ArrayRef<const uint8_t> kNopCode; + static const uint8_t kUnpatchedPcRelativeRawCode[]; + static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode; + static const uint32_t kPcInsnOffset; // Branches within range [-256, 256) can be created from these by adding the low 8 bits. static constexpr uint32_t kBlPlus0 = 0xf000f800; @@ -123,47 +126,9 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { return result; } - void TestDexCachereference(uint32_t dex_cache_arrays_begin, uint32_t element_offset) { - dex_cache_arrays_begin_ = dex_cache_arrays_begin; - static const uint8_t raw_code[] = { - 0x40, 0xf2, 0x00, 0x00, // MOVW r0, #0 (placeholder) - 0xc0, 0xf2, 0x00, 0x00, // MOVT r0, #0 (placeholder) - 0x78, 0x44, // ADD r0, pc - }; - constexpr uint32_t pc_insn_offset = 8u; - const ArrayRef<const uint8_t> code(raw_code); - LinkerPatch patches[] = { - LinkerPatch::DexCacheArrayPatch(0u, nullptr, pc_insn_offset, element_offset), - LinkerPatch::DexCacheArrayPatch(4u, nullptr, pc_insn_offset, element_offset), - }; - AddCompiledMethod(MethodRef(1u), code, ArrayRef<const LinkerPatch>(patches)); - Link(); - - uint32_t method1_offset = GetMethodOffset(1u); - uint32_t pc_base_offset = method1_offset + pc_insn_offset + 4u /* PC adjustment */; - uint32_t diff = dex_cache_arrays_begin_ + element_offset - pc_base_offset; - // Distribute the bits of the diff between the MOVW and MOVT: - uint32_t diffw = diff & 0xffffu; - uint32_t difft = diff >> 16; - uint32_t movw = 0xf2400000u | // MOVW r0, #0 (placeholder), - ((diffw & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19, - ((diffw & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26, - ((diffw & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14, - ((diffw & 0x00ffu)); // keep imm8 at bits 0-7. - uint32_t movt = 0xf2c00000u | // MOVT r0, #0 (placeholder), - ((difft & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19, - ((difft & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26, - ((difft & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14, - ((difft & 0x00ffu)); // keep imm8 at bits 0-7. - const uint8_t expected_code[] = { - static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24), - static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8), - static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24), - static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8), - 0x78, 0x44, - }; - EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); - } + void TestDexCacheReference(uint32_t dex_cache_arrays_begin, uint32_t element_offset); + void TestStringReference(uint32_t string_offset); + void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset); }; const uint8_t Thumb2RelativePatcherTest::kCallRawCode[] = { @@ -178,6 +143,67 @@ const uint8_t Thumb2RelativePatcherTest::kNopRawCode[] = { const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kNopCode(kNopRawCode); +const uint8_t Thumb2RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = { + 0x40, 0xf2, 0x00, 0x00, // MOVW r0, #0 (placeholder) + 0xc0, 0xf2, 0x00, 0x00, // MOVT r0, #0 (placeholder) + 0x78, 0x44, // ADD r0, pc +}; +const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kUnpatchedPcRelativeCode( + kUnpatchedPcRelativeRawCode); +const uint32_t Thumb2RelativePatcherTest::kPcInsnOffset = 8u; + +void Thumb2RelativePatcherTest::TestDexCacheReference(uint32_t dex_cache_arrays_begin, + uint32_t element_offset) { + dex_cache_arrays_begin_ = dex_cache_arrays_begin; + LinkerPatch patches[] = { + LinkerPatch::DexCacheArrayPatch(0u, nullptr, kPcInsnOffset, element_offset), + LinkerPatch::DexCacheArrayPatch(4u, nullptr, kPcInsnOffset, element_offset), + }; + CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), + dex_cache_arrays_begin_ + element_offset); +} + +void Thumb2RelativePatcherTest::TestStringReference(uint32_t string_offset) { + constexpr uint32_t kStringIndex = 1u; + string_index_to_offset_map_.Put(kStringIndex, string_offset); + LinkerPatch patches[] = { + LinkerPatch::RelativeStringPatch(0u, nullptr, kPcInsnOffset, kStringIndex), + LinkerPatch::RelativeStringPatch(4u, nullptr, kPcInsnOffset, kStringIndex), + }; + CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset); +} + +void Thumb2RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, + uint32_t target_offset) { + AddCompiledMethod(MethodRef(1u), kUnpatchedPcRelativeCode, ArrayRef<const LinkerPatch>(patches)); + Link(); + + uint32_t method1_offset = GetMethodOffset(1u); + uint32_t pc_base_offset = method1_offset + kPcInsnOffset + 4u /* PC adjustment */; + uint32_t diff = target_offset - pc_base_offset; + // Distribute the bits of the diff between the MOVW and MOVT: + uint32_t diffw = diff & 0xffffu; + uint32_t difft = diff >> 16; + uint32_t movw = 0xf2400000u | // MOVW r0, #0 (placeholder), + ((diffw & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19, + ((diffw & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26, + ((diffw & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14, + ((diffw & 0x00ffu)); // keep imm8 at bits 0-7. + uint32_t movt = 0xf2c00000u | // MOVT r0, #0 (placeholder), + ((difft & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19, + ((difft & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26, + ((difft & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14, + ((difft & 0x00ffu)); // keep imm8 at bits 0-7. + const uint8_t expected_code[] = { + static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24), + static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8), + static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24), + static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8), + 0x78, 0x44, + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); +} + TEST_F(Thumb2RelativePatcherTest, CallSelf) { LinkerPatch patches[] = { LinkerPatch::RelativeCodePatch(0u, nullptr, 1u), @@ -366,23 +392,43 @@ TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarBefore) { EXPECT_TRUE(CheckThunk(thunk_offset)); } -TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceImm8) { - TestDexCachereference(0x00ff0000u, 0x00fcu); +TEST_F(Thumb2RelativePatcherTest, DexCacheReference1) { + TestDexCacheReference(0x00ff0000u, 0x00fcu); + ASSERT_LT(GetMethodOffset(1u), 0xfcu); +} + +TEST_F(Thumb2RelativePatcherTest, DexCacheReference2) { + TestDexCacheReference(0x02ff0000u, 0x05fcu); + ASSERT_LT(GetMethodOffset(1u), 0xfcu); +} + +TEST_F(Thumb2RelativePatcherTest, DexCacheReference3) { + TestDexCacheReference(0x08ff0000u, 0x08fcu); + ASSERT_LT(GetMethodOffset(1u), 0xfcu); +} + +TEST_F(Thumb2RelativePatcherTest, DexCacheReference4) { + TestDexCacheReference(0xd0ff0000u, 0x60fcu); + ASSERT_LT(GetMethodOffset(1u), 0xfcu); +} + +TEST_F(Thumb2RelativePatcherTest, StringReference1) { + TestStringReference(0x00ff00fcu); ASSERT_LT(GetMethodOffset(1u), 0xfcu); } -TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceImm3) { - TestDexCachereference(0x02ff0000u, 0x05fcu); +TEST_F(Thumb2RelativePatcherTest, StringReference2) { + TestStringReference(0x02ff05fcu); ASSERT_LT(GetMethodOffset(1u), 0xfcu); } -TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceImm) { - TestDexCachereference(0x08ff0000u, 0x08fcu); +TEST_F(Thumb2RelativePatcherTest, StringReference3) { + TestStringReference(0x08ff08fcu); ASSERT_LT(GetMethodOffset(1u), 0xfcu); } -TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceimm4) { - TestDexCachereference(0xd0ff0000u, 0x60fcu); +TEST_F(Thumb2RelativePatcherTest, StringReference4) { + TestStringReference(0xd0ff60fcu); ASSERT_LT(GetMethodOffset(1u), 0xfcu); } diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index a81c85c707..e3e31213ba 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -28,6 +28,17 @@ namespace art { namespace linker { +namespace { + +inline bool IsAdrpPatch(const LinkerPatch& patch) { + LinkerPatch::Type type = patch.GetType(); + return + (type == LinkerPatch::Type::kStringRelative || type == LinkerPatch::Type::kDexCacheArray) && + patch.LiteralOffset() == patch.PcInsnOffset(); +} + +} // anonymous namespace + Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, const Arm64InstructionSetFeatures* features) : ArmBaseRelativePatcher(provider, kArm64, CompileThunkCode(), @@ -61,8 +72,7 @@ uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, size_t num_adrp = 0u; DCHECK(compiled_method != nullptr); for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.Type() == kLinkerPatchDexCacheArray && - patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch + if (IsAdrpPatch(patch)) { ++num_adrp; } } @@ -78,8 +88,7 @@ uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); DCHECK(compiled_method != nullptr); for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.Type() == kLinkerPatchDexCacheArray && - patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch + if (IsAdrpPatch(patch)) { uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); @@ -151,10 +160,10 @@ void Arm64RelativePatcher::PatchCall(std::vector<uint8_t>* code, SetInsn(code, literal_offset, insn); } -void Arm64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) { +void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) { DCHECK_EQ(patch_offset & 3u, 0u); DCHECK_EQ(target_offset & 3u, 0u); uint32_t literal_offset = patch.LiteralOffset(); @@ -199,8 +208,15 @@ void Arm64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, // Write the new ADRP (or B to the erratum 843419 thunk). SetInsn(code, literal_offset, insn); } else { - // LDR 32-bit or 64-bit with imm12 == 0 (unset). - DCHECK_EQ(insn & 0xbffffc00, 0xb9400000) << insn; + if ((insn & 0xfffffc00) == 0x91000000) { + // ADD immediate, 64-bit with imm12 == 0 (unset). + DCHECK(patch.GetType() == LinkerPatch::Type::kStringRelative) << patch.GetType(); + shift = 0u; // No shift for ADD. + } else { + // LDR 32-bit or 64-bit with imm12 == 0 (unset). + DCHECK(patch.GetType() == LinkerPatch::Type::kDexCacheArray) << patch.GetType(); + DCHECK_EQ(insn & 0xbffffc00, 0xb9400000) << std::hex << insn; + } if (kIsDebugBuild) { uint32_t adrp = GetInsn(code, pc_insn_offset); if ((adrp & 0x9f000000u) != 0x90000000u) { @@ -263,7 +279,7 @@ bool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, DCHECK_EQ(patch_offset & 0x3u, 0u); if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc uint32_t adrp = GetInsn(code, literal_offset); - DCHECK_EQ(adrp & 0xff000000, 0x90000000); + DCHECK_EQ(adrp & 0x9f000000, 0x90000000); uint32_t next_offset = patch_offset + 4u; uint32_t next_insn = GetInsn(code, literal_offset + 4u); @@ -277,6 +293,15 @@ bool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, return false; } + // And since LinkerPatch::Type::kStringRelative is using the result of the ADRP + // for an ADD immediate, check for that as well. We generalize a bit to include + // ADD/ADDS/SUB/SUBS immediate that either uses the ADRP destination or stores + // the result to a different register. + if ((next_insn & 0x1f000000) == 0x11000000 && + ((((next_insn >> 5) ^ adrp) & 0x1f) == 0 || ((next_insn ^ adrp) & 0x1f) != 0)) { + return false; + } + // LDR <Wt>, <label> is always aligned and thus it doesn't cause boundary crossing. if ((next_insn & 0xff000000) == 0x18000000) { return false; diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h index f9b76e6250..48ad1059b0 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.h +++ b/compiler/linker/arm64/relative_patcher_arm64.h @@ -37,10 +37,10 @@ class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { uint32_t literal_offset, uint32_t patch_offset, uint32_t target_offset) OVERRIDE; - void PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) OVERRIDE; + void PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) OVERRIDE; private: static std::vector<uint8_t> CompileThunkCode(); diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc index 0bfef5e6d3..09729fdf96 100644 --- a/compiler/linker/arm64/relative_patcher_arm64_test.cc +++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc @@ -40,6 +40,15 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { static constexpr uint32_t kBlPlusMax = 0x95ffffffu; static constexpr uint32_t kBlMinusMax = 0x96000000u; + // LDR immediate, 32-bit. + static constexpr uint32_t kLdrWInsn = 0xb9400000u; + + // ADD/ADDS/SUB/SUBS immediate, 64-bit. + static constexpr uint32_t kAddXInsn = 0x91000000u; + static constexpr uint32_t kAddsXInsn = 0xb1000000u; + static constexpr uint32_t kSubXInsn = 0xd1000000u; + static constexpr uint32_t kSubsXInsn = 0xf1000000u; + // LDUR x2, [sp, #4], i.e. unaligned load crossing 64-bit boundary (assuming aligned sp). static constexpr uint32_t kLdurInsn = 0xf840405fu; @@ -109,7 +118,7 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { uint32_t GetMethodOffset(uint32_t method_idx) { auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx)); CHECK(result.first); - CHECK_EQ(result.second & 3u, 0u); + CHECK_ALIGNED(result.second, 4u); return result.second; } @@ -147,20 +156,29 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { return result; } - std::vector<uint8_t> GenNopsAndAdrpLdr(size_t num_nops, - uint32_t method_offset, uint32_t target_offset) { + std::vector<uint8_t> GenNopsAndAdrpAndUse(size_t num_nops, + uint32_t method_offset, + uint32_t target_offset, + uint32_t use_insn) { std::vector<uint8_t> result; result.reserve(num_nops * 4u + 8u); for (size_t i = 0; i != num_nops; ++i) { result.insert(result.end(), kNopCode.begin(), kNopCode.end()); } - DCHECK_EQ(method_offset & 3u, 0u); - DCHECK_EQ(target_offset & 3u, 0u); + CHECK_ALIGNED(method_offset, 4u); + CHECK_ALIGNED(target_offset, 4u); uint32_t adrp_offset = method_offset + num_nops * 4u; uint32_t disp = target_offset - (adrp_offset & ~0xfffu); - DCHECK_EQ(disp & 3u, 0u); - uint32_t ldr = 0xb9400001 | // LDR w1, [x0, #(imm12 * 2)] - ((disp & 0xfffu) << (10 - 2)); // imm12 = ((disp & 0xfffu) >> 2) is at bit 10. + if (use_insn == kLdrWInsn) { + DCHECK_ALIGNED(disp, 1u << 2); + use_insn |= 1 | // LDR x1, [x0, #(imm12 << 2)] + ((disp & 0xfffu) << (10 - 2)); // imm12 = ((disp & 0xfffu) >> 2) is at bit 10. + } else if (use_insn == kAddXInsn) { + use_insn |= 1 | // ADD x1, x0, #imm + (disp & 0xfffu) << 10; // imm12 = (disp & 0xfffu) is at bit 10. + } else { + LOG(FATAL) << "Unexpected instruction: 0x" << std::hex << use_insn; + } uint32_t adrp = 0x90000000 | // ADRP x0, +SignExtend(immhi:immlo:Zeros(12), 64) ((disp & 0x3000u) << (29 - 12)) | // immlo = ((disp & 0x3000u) >> 12) is at bit 29, ((disp & 0xffffc000) >> (14 - 5)) | // immhi = (disp >> 14) is at bit 5, @@ -170,13 +188,19 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { result.push_back(static_cast<uint8_t>(adrp >> 8)); result.push_back(static_cast<uint8_t>(adrp >> 16)); result.push_back(static_cast<uint8_t>(adrp >> 24)); - result.push_back(static_cast<uint8_t>(ldr)); - result.push_back(static_cast<uint8_t>(ldr >> 8)); - result.push_back(static_cast<uint8_t>(ldr >> 16)); - result.push_back(static_cast<uint8_t>(ldr >> 24)); + result.push_back(static_cast<uint8_t>(use_insn)); + result.push_back(static_cast<uint8_t>(use_insn >> 8)); + result.push_back(static_cast<uint8_t>(use_insn >> 16)); + result.push_back(static_cast<uint8_t>(use_insn >> 24)); return result; } + std::vector<uint8_t> GenNopsAndAdrpLdr(size_t num_nops, + uint32_t method_offset, + uint32_t target_offset) { + return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kLdrWInsn); + } + void TestNopsAdrpLdr(size_t num_nops, uint32_t dex_cache_arrays_begin, uint32_t element_offset) { dex_cache_arrays_begin_ = dex_cache_arrays_begin; auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u); // Unpatched. @@ -184,7 +208,8 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { LinkerPatch::DexCacheArrayPatch(num_nops * 4u , nullptr, num_nops * 4u, element_offset), LinkerPatch::DexCacheArrayPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, element_offset), }; - AddCompiledMethod(MethodRef(1u), ArrayRef<const uint8_t>(code), + AddCompiledMethod(MethodRef(1u), + ArrayRef<const uint8_t>(code), ArrayRef<const LinkerPatch>(patches)); Link(); @@ -194,6 +219,30 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } + std::vector<uint8_t> GenNopsAndAdrpAdd(size_t num_nops, + uint32_t method_offset, + uint32_t target_offset) { + return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kAddXInsn); + } + + void TestNopsAdrpAdd(size_t num_nops, uint32_t string_offset) { + constexpr uint32_t kStringIndex = 1u; + string_index_to_offset_map_.Put(kStringIndex, string_offset); + auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u); // Unpatched. + LinkerPatch patches[] = { + LinkerPatch::RelativeStringPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex), + LinkerPatch::RelativeStringPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, kStringIndex), + }; + AddCompiledMethod(MethodRef(1u), + ArrayRef<const uint8_t>(code), + ArrayRef<const LinkerPatch>(patches)); + Link(); + + uint32_t method1_offset = GetMethodOffset(1u); + auto expected_code = GenNopsAndAdrpAdd(num_nops, method1_offset, string_offset); + EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); + } + void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) { CHECK_LE(pos, code->size()); const uint8_t insn_code[] = { @@ -204,8 +253,10 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code)); } - void PrepareNopsAdrpInsn2Ldr(size_t num_nops, uint32_t insn2, - uint32_t dex_cache_arrays_begin, uint32_t element_offset) { + void PrepareNopsAdrpInsn2Ldr(size_t num_nops, + uint32_t insn2, + uint32_t dex_cache_arrays_begin, + uint32_t element_offset) { dex_cache_arrays_begin_ = dex_cache_arrays_begin; auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u); // Unpatched. InsertInsn(&code, num_nops * 4u + 4u, insn2); @@ -213,26 +264,41 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { LinkerPatch::DexCacheArrayPatch(num_nops * 4u , nullptr, num_nops * 4u, element_offset), LinkerPatch::DexCacheArrayPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, element_offset), }; - AddCompiledMethod(MethodRef(1u), ArrayRef<const uint8_t>(code), + AddCompiledMethod(MethodRef(1u), + ArrayRef<const uint8_t>(code), ArrayRef<const LinkerPatch>(patches)); Link(); } - void TestNopsAdrpInsn2Ldr(size_t num_nops, uint32_t insn2, - uint32_t dex_cache_arrays_begin, uint32_t element_offset) { - PrepareNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset); + void PrepareNopsAdrpInsn2Add(size_t num_nops, uint32_t insn2, uint32_t string_offset) { + constexpr uint32_t kStringIndex = 1u; + string_index_to_offset_map_.Put(kStringIndex, string_offset); + auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u); // Unpatched. + InsertInsn(&code, num_nops * 4u + 4u, insn2); + LinkerPatch patches[] = { + LinkerPatch::RelativeStringPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex), + LinkerPatch::RelativeStringPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, kStringIndex), + }; + AddCompiledMethod(MethodRef(1u), + ArrayRef<const uint8_t>(code), + ArrayRef<const LinkerPatch>(patches)); + Link(); + } + void TestNopsAdrpInsn2AndUse(size_t num_nops, + uint32_t insn2, + uint32_t target_offset, + uint32_t use_insn) { uint32_t method1_offset = GetMethodOffset(1u); - uint32_t target_offset = dex_cache_arrays_begin_ + element_offset; - auto expected_code = GenNopsAndAdrpLdr(num_nops, method1_offset, target_offset); + auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn); InsertInsn(&expected_code, num_nops * 4u + 4u, insn2); EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } - void TestNopsAdrpInsn2LdrHasThunk(size_t num_nops, uint32_t insn2, - uint32_t dex_cache_arrays_begin, uint32_t element_offset) { - PrepareNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset); - + void TestNopsAdrpInsn2AndUseHasThunk(size_t num_nops, + uint32_t insn2, + uint32_t target_offset, + uint32_t use_insn) { uint32_t method1_offset = GetMethodOffset(1u); CHECK(!compiled_method_refs_.empty()); CHECK_EQ(compiled_method_refs_[0].dex_method_index, 1u); @@ -240,13 +306,12 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { uint32_t method1_size = compiled_methods_[0]->GetQuickCode().size(); uint32_t thunk_offset = CompiledCode::AlignCode(method1_offset + method1_size, kArm64); uint32_t b_diff = thunk_offset - (method1_offset + num_nops * 4u); - ASSERT_EQ(b_diff & 3u, 0u); + CHECK_ALIGNED(b_diff, 4u); ASSERT_LT(b_diff, 128 * MB); uint32_t b_out = kBPlus0 + ((b_diff >> 2) & 0x03ffffffu); uint32_t b_in = kBPlus0 + ((-b_diff >> 2) & 0x03ffffffu); - uint32_t target_offset = dex_cache_arrays_begin_ + element_offset; - auto expected_code = GenNopsAndAdrpLdr(num_nops, method1_offset, target_offset); + auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn); InsertInsn(&expected_code, num_nops * 4u + 4u, insn2); // Replace adrp with bl. expected_code.erase(expected_code.begin() + num_nops * 4u, @@ -270,29 +335,39 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { } } - void TestAdrpInsn2Ldr(uint32_t insn2, uint32_t adrp_offset, bool has_thunk, - uint32_t dex_cache_arrays_begin, uint32_t element_offset) { + void TestAdrpInsn2Ldr(uint32_t insn2, + uint32_t adrp_offset, + bool has_thunk, + uint32_t dex_cache_arrays_begin, + uint32_t element_offset) { uint32_t method1_offset = CompiledCode::AlignCode(kTrampolineSize, kArm64) + sizeof(OatQuickMethodHeader); ASSERT_LT(method1_offset, adrp_offset); - ASSERT_EQ(adrp_offset & 3u, 0u); + CHECK_ALIGNED(adrp_offset, 4u); uint32_t num_nops = (adrp_offset - method1_offset) / 4u; + PrepareNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset); + uint32_t target_offset = dex_cache_arrays_begin_ + element_offset; if (has_thunk) { - TestNopsAdrpInsn2LdrHasThunk(num_nops, insn2, dex_cache_arrays_begin, element_offset); + TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, target_offset, kLdrWInsn); } else { - TestNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset); + TestNopsAdrpInsn2AndUse(num_nops, insn2, target_offset, kLdrWInsn); } ASSERT_EQ(method1_offset, GetMethodOffset(1u)); // If this fails, num_nops is wrong. } - void TestAdrpLdurLdr(uint32_t adrp_offset, bool has_thunk, - uint32_t dex_cache_arrays_begin, uint32_t element_offset) { + void TestAdrpLdurLdr(uint32_t adrp_offset, + bool has_thunk, + uint32_t dex_cache_arrays_begin, + uint32_t element_offset) { TestAdrpInsn2Ldr(kLdurInsn, adrp_offset, has_thunk, dex_cache_arrays_begin, element_offset); } - void TestAdrpLdrPcRelLdr(uint32_t pcrel_ldr_insn, int32_t pcrel_disp, - uint32_t adrp_offset, bool has_thunk, - uint32_t dex_cache_arrays_begin, uint32_t element_offset) { + void TestAdrpLdrPcRelLdr(uint32_t pcrel_ldr_insn, + int32_t pcrel_disp, + uint32_t adrp_offset, + bool has_thunk, + uint32_t dex_cache_arrays_begin, + uint32_t element_offset) { ASSERT_LT(pcrel_disp, 0x100000); ASSERT_GE(pcrel_disp, -0x100000); ASSERT_EQ(pcrel_disp & 0x3, 0); @@ -300,13 +375,60 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, dex_cache_arrays_begin, element_offset); } - void TestAdrpLdrSpRelLdr(uint32_t sprel_ldr_insn, uint32_t sprel_disp_in_load_units, - uint32_t adrp_offset, bool has_thunk, - uint32_t dex_cache_arrays_begin, uint32_t element_offset) { + void TestAdrpLdrSpRelLdr(uint32_t sprel_ldr_insn, + uint32_t sprel_disp_in_load_units, + uint32_t adrp_offset, + bool has_thunk, + uint32_t dex_cache_arrays_begin, + uint32_t element_offset) { ASSERT_LT(sprel_disp_in_load_units, 0x1000u); uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10); TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, dex_cache_arrays_begin, element_offset); } + + void TestAdrpInsn2Add(uint32_t insn2, + uint32_t adrp_offset, + bool has_thunk, + uint32_t string_offset) { + uint32_t method1_offset = + CompiledCode::AlignCode(kTrampolineSize, kArm64) + sizeof(OatQuickMethodHeader); + ASSERT_LT(method1_offset, adrp_offset); + CHECK_ALIGNED(adrp_offset, 4u); + uint32_t num_nops = (adrp_offset - method1_offset) / 4u; + PrepareNopsAdrpInsn2Add(num_nops, insn2, string_offset); + if (has_thunk) { + TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, string_offset, kAddXInsn); + } else { + TestNopsAdrpInsn2AndUse(num_nops, insn2, string_offset, kAddXInsn); + } + ASSERT_EQ(method1_offset, GetMethodOffset(1u)); // If this fails, num_nops is wrong. + } + + void TestAdrpLdurAdd(uint32_t adrp_offset, bool has_thunk, uint32_t string_offset) { + TestAdrpInsn2Add(kLdurInsn, adrp_offset, has_thunk, string_offset); + } + + void TestAdrpLdrPcRelAdd(uint32_t pcrel_ldr_insn, + int32_t pcrel_disp, + uint32_t adrp_offset, + bool has_thunk, + uint32_t string_offset) { + ASSERT_LT(pcrel_disp, 0x100000); + ASSERT_GE(pcrel_disp, -0x100000); + ASSERT_EQ(pcrel_disp & 0x3, 0); + uint32_t insn2 = pcrel_ldr_insn | (((static_cast<uint32_t>(pcrel_disp) >> 2) & 0x7ffffu) << 5); + TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset); + } + + void TestAdrpLdrSpRelAdd(uint32_t sprel_ldr_insn, + uint32_t sprel_disp_in_load_units, + uint32_t adrp_offset, + bool has_thunk, + uint32_t string_offset) { + ASSERT_LT(sprel_disp_in_load_units, 0x1000u); + uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10); + TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset); + } }; const uint8_t Arm64RelativePatcherTest::kCallRawCode[] = { @@ -358,14 +480,14 @@ TEST_F(Arm64RelativePatcherTestDefault, CallOther) { uint32_t method1_offset = GetMethodOffset(1u); uint32_t method2_offset = GetMethodOffset(2u); uint32_t diff_after = method2_offset - method1_offset; - ASSERT_EQ(diff_after & 3u, 0u); + CHECK_ALIGNED(diff_after, 4u); ASSERT_LT(diff_after >> 2, 1u << 8); // Simple encoding, (diff_after >> 2) fits into 8 bits. static const uint8_t method1_expected_code[] = { static_cast<uint8_t>(diff_after >> 2), 0x00, 0x00, 0x94 }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code))); uint32_t diff_before = method1_offset - method2_offset; - ASSERT_EQ(diff_before & 3u, 0u); + CHECK_ALIGNED(diff_before, 4u); ASSERT_GE(diff_before, -1u << 27); auto method2_expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff_before >> 2) & 0x03ffffffu)); EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code))); @@ -411,7 +533,7 @@ TEST_F(Arm64RelativePatcherTestDefault, CallTrampolineTooFar) { uint32_t thunk_offset = CompiledCode::AlignCode(last_method_offset + last_method_code.size(), kArm64); uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method); - ASSERT_EQ(diff & 3u, 0u); + CHECK_ALIGNED(diff, 4u); ASSERT_LT(diff, 128 * MB); auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2)); EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx), @@ -497,7 +619,7 @@ TEST_F(Arm64RelativePatcherTestDefault, CallOtherJustTooFarAfter) { uint32_t thunk_offset = last_method_header_offset - CompiledCode::AlignCode(ThunkSize(), kArm64); ASSERT_TRUE(IsAligned<kArm64Alignment>(thunk_offset)); uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1); - ASSERT_EQ(diff & 3u, 0u); + CHECK_ALIGNED(diff, 4u); ASSERT_LT(diff, 128 * MB); auto expected_code = GenNopsAndBl(0u, kBlPlus0 | (diff >> 2)); EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); @@ -527,7 +649,7 @@ TEST_F(Arm64RelativePatcherTestDefault, CallOtherJustTooFarBefore) { uint32_t thunk_offset = CompiledCode::AlignCode(last_method_offset + last_method_code.size(), kArm64); uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method); - ASSERT_EQ(diff & 3u, 0u); + CHECK_ALIGNED(diff, 4u); ASSERT_LT(diff, 128 * MB); auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2)); EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx), @@ -551,74 +673,158 @@ TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference4) { TestNopsAdrpLdr(0u, 0x12345000u, 0x4000u); } -TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0xff4) { - TestAdrpLdurLdr(0xff4u, false, 0x12345678u, 0x1234u); +TEST_F(Arm64RelativePatcherTestDefault, StringReference1) { + TestNopsAdrpAdd(0u, 0x12345678u); } -TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0xff8) { - TestAdrpLdurLdr(0xff8u, true, 0x12345678u, 0x1234u); +TEST_F(Arm64RelativePatcherTestDefault, StringReference2) { + TestNopsAdrpAdd(0u, -0x12345678u); } -TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0xffc) { - TestAdrpLdurLdr(0xffcu, true, 0x12345678u, 0x1234u); +TEST_F(Arm64RelativePatcherTestDefault, StringReference3) { + TestNopsAdrpAdd(0u, 0x12345000u); } -TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0x1000) { - TestAdrpLdurLdr(0x1000u, false, 0x12345678u, 0x1234u); +TEST_F(Arm64RelativePatcherTestDefault, StringReference4) { + TestNopsAdrpAdd(0u, 0x12345ffcu); } -TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0xff4) { - TestAdrpLdurLdr(0xff4u, false, 0x12345678u, 0x1234u); -} +#define TEST_FOR_OFFSETS(test, disp1, disp2) \ + test(0xff4u, disp1) test(0xff8u, disp1) test(0xffcu, disp1) test(0x1000u, disp1) \ + test(0xff4u, disp2) test(0xff8u, disp2) test(0xffcu, disp2) test(0x1000u, disp2) -TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0xff8) { - TestAdrpLdurLdr(0xff8u, false, 0x12345678u, 0x1234u); -} +#define DEFAULT_LDUR_LDR_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## Ldur ## disp) { \ + bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu); \ + TestAdrpLdurLdr(adrp_offset, has_thunk, 0x12345678u, disp); \ + } -TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0xffc) { - TestAdrpLdurLdr(0xffcu, false, 0x12345678u, 0x1234u); -} +TEST_FOR_OFFSETS(DEFAULT_LDUR_LDR_TEST, 0x1234, 0x1238) -TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0x1000) { - TestAdrpLdurLdr(0x1000u, false, 0x12345678u, 0x1234u); -} +#define DENVER64_LDUR_LDR_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference ## adrp_offset ## Ldur ## disp) { \ + TestAdrpLdurLdr(adrp_offset, false, 0x12345678u, disp); \ + } -#define TEST_FOR_OFFSETS(test, disp1, disp2) \ - test(0xff4u, disp1) test(0xff8u, disp1) test(0xffcu, disp1) test(0x1000u, disp1) \ - test(0xff4u, disp2) test(0xff8u, disp2) test(0xffcu, disp2) test(0x1000u, disp2) +TEST_FOR_OFFSETS(DENVER64_LDUR_LDR_TEST, 0x1234, 0x1238) // LDR <Wt>, <label> is always aligned. We should never have to use a fixup. -#define LDRW_PCREL_TEST(adrp_offset, disp) \ +#define LDRW_PCREL_LDR_TEST(adrp_offset, disp) \ TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## WPcRel ## disp) { \ TestAdrpLdrPcRelLdr(kLdrWPcRelInsn, disp, adrp_offset, false, 0x12345678u, 0x1234u); \ } -TEST_FOR_OFFSETS(LDRW_PCREL_TEST, 0x1234, 0x1238) +TEST_FOR_OFFSETS(LDRW_PCREL_LDR_TEST, 0x1234, 0x1238) // LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8. -#define LDRX_PCREL_TEST(adrp_offset, disp) \ +#define LDRX_PCREL_LDR_TEST(adrp_offset, disp) \ TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## XPcRel ## disp) { \ - bool unaligned = ((adrp_offset + 4u + static_cast<uint32_t>(disp)) & 7u) != 0; \ + bool unaligned = !IsAligned<8u>(adrp_offset + 4u + static_cast<uint32_t>(disp)); \ bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu) && unaligned; \ TestAdrpLdrPcRelLdr(kLdrXPcRelInsn, disp, adrp_offset, has_thunk, 0x12345678u, 0x1234u); \ } -TEST_FOR_OFFSETS(LDRX_PCREL_TEST, 0x1234, 0x1238) +TEST_FOR_OFFSETS(LDRX_PCREL_LDR_TEST, 0x1234, 0x1238) // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed. -#define LDRW_SPREL_TEST(adrp_offset, disp) \ +#define LDRW_SPREL_LDR_TEST(adrp_offset, disp) \ TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## WSpRel ## disp) { \ TestAdrpLdrSpRelLdr(kLdrWSpRelInsn, disp >> 2, adrp_offset, false, 0x12345678u, 0x1234u); \ } -TEST_FOR_OFFSETS(LDRW_SPREL_TEST, 0, 4) +TEST_FOR_OFFSETS(LDRW_SPREL_LDR_TEST, 0, 4) -#define LDRX_SPREL_TEST(adrp_offset, disp) \ +#define LDRX_SPREL_LDR_TEST(adrp_offset, disp) \ TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## XSpRel ## disp) { \ TestAdrpLdrSpRelLdr(kLdrXSpRelInsn, disp >> 3, adrp_offset, false, 0x12345678u, 0x1234u); \ } -TEST_FOR_OFFSETS(LDRX_SPREL_TEST, 0, 8) +TEST_FOR_OFFSETS(LDRX_SPREL_LDR_TEST, 0, 8) + +#define DEFAULT_LDUR_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## Ldur ## disp) { \ + bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu); \ + TestAdrpLdurAdd(adrp_offset, has_thunk, disp); \ + } + +TEST_FOR_OFFSETS(DEFAULT_LDUR_ADD_TEST, 0x12345678, 0xffffc840) + +#define DENVER64_LDUR_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDenver64, StringReference ## adrp_offset ## Ldur ## disp) { \ + TestAdrpLdurAdd(adrp_offset, false, disp); \ + } + +TEST_FOR_OFFSETS(DENVER64_LDUR_ADD_TEST, 0x12345678, 0xffffc840) + +#define DEFAULT_SUBX3X2_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## SubX3X2 ## disp) { \ + /* SUB unrelated to "ADRP x0, addr". */ \ + uint32_t sub = kSubXInsn | (100 << 10) | (2u << 5) | 3u; /* SUB x3, x2, #100 */ \ + TestAdrpInsn2Add(sub, adrp_offset, false, disp); \ + } + +TEST_FOR_OFFSETS(DEFAULT_SUBX3X2_ADD_TEST, 0x12345678, 0xffffc840) + +#define DEFAULT_SUBSX3X0_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## SubsX3X0 ## disp) { \ + /* SUBS that uses the result of "ADRP x0, addr". */ \ + uint32_t subs = kSubsXInsn | (100 << 10) | (0u << 5) | 3u; /* SUBS x3, x0, #100 */ \ + TestAdrpInsn2Add(subs, adrp_offset, false, disp); \ + } + +TEST_FOR_OFFSETS(DEFAULT_SUBSX3X0_ADD_TEST, 0x12345678, 0xffffc840) + +#define DEFAULT_ADDX0X0_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## AddX0X0 ## disp) { \ + /* ADD that uses the result register of "ADRP x0, addr" as both source and destination. */ \ + uint32_t add = kSubXInsn | (100 << 10) | (0u << 5) | 0u; /* ADD x0, x0, #100 */ \ + TestAdrpInsn2Add(add, adrp_offset, false, disp); \ + } + +TEST_FOR_OFFSETS(DEFAULT_ADDX0X0_ADD_TEST, 0x12345678, 0xffffc840) + +#define DEFAULT_ADDSX0X2_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## AddsX0X2 ## disp) { \ + /* ADDS that does not use the result of "ADRP x0, addr" but overwrites that register. */ \ + uint32_t adds = kAddsXInsn | (100 << 10) | (2u << 5) | 0u; /* ADDS x0, x2, #100 */ \ + bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu); \ + TestAdrpInsn2Add(adds, adrp_offset, has_thunk, disp); \ + } + +TEST_FOR_OFFSETS(DEFAULT_ADDSX0X2_ADD_TEST, 0x12345678, 0xffffc840) + +// LDR <Wt>, <label> is always aligned. We should never have to use a fixup. +#define LDRW_PCREL_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## WPcRel ## disp) { \ + TestAdrpLdrPcRelAdd(kLdrWPcRelInsn, disp, adrp_offset, false, 0x12345678u); \ + } + +TEST_FOR_OFFSETS(LDRW_PCREL_ADD_TEST, 0x1234, 0x1238) + +// LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8. +#define LDRX_PCREL_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## XPcRel ## disp) { \ + bool unaligned = !IsAligned<8u>(adrp_offset + 4u + static_cast<uint32_t>(disp)); \ + bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu) && unaligned; \ + TestAdrpLdrPcRelAdd(kLdrXPcRelInsn, disp, adrp_offset, has_thunk, 0x12345678u); \ + } + +TEST_FOR_OFFSETS(LDRX_PCREL_ADD_TEST, 0x1234, 0x1238) + +// LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed. +#define LDRW_SPREL_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## WSpRel ## disp) { \ + TestAdrpLdrSpRelAdd(kLdrWSpRelInsn, disp >> 2, adrp_offset, false, 0x12345678u); \ + } + +TEST_FOR_OFFSETS(LDRW_SPREL_ADD_TEST, 0, 4) + +#define LDRX_SPREL_ADD_TEST(adrp_offset, disp) \ + TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## XSpRel ## disp) { \ + TestAdrpLdrSpRelAdd(kLdrXSpRelInsn, disp >> 3, adrp_offset, false, 0x12345678u); \ + } + +TEST_FOR_OFFSETS(LDRX_SPREL_ADD_TEST, 0, 8) } // namespace linker } // namespace art diff --git a/compiler/linker/multi_oat_relative_patcher.h b/compiler/linker/multi_oat_relative_patcher.h index 1727d529fc..dbda03fd3b 100644 --- a/compiler/linker/multi_oat_relative_patcher.h +++ b/compiler/linker/multi_oat_relative_patcher.h @@ -103,13 +103,13 @@ class MultiOatRelativePatcher FINAL { } // Wrapper around RelativePatcher::PatchDexCacheReference(), doing offset adjustment. - void PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) { + void PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) { patch_offset += adjustment_; target_offset += adjustment_; - relative_patcher_->PatchDexCacheReference(code, patch, patch_offset, target_offset); + relative_patcher_->PatchPcRelativeReference(code, patch, patch_offset, target_offset); } // Wrappers around RelativePatcher for statistics retrieval. diff --git a/compiler/linker/multi_oat_relative_patcher_test.cc b/compiler/linker/multi_oat_relative_patcher_test.cc index 792cdfe8e9..92a96a0bd3 100644 --- a/compiler/linker/multi_oat_relative_patcher_test.cc +++ b/compiler/linker/multi_oat_relative_patcher_test.cc @@ -86,10 +86,10 @@ class MultiOatRelativePatcherTest : public testing::Test { last_target_offset_ = target_offset; } - void PatchDexCacheReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) OVERRIDE { + void PatchPcRelativeReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) OVERRIDE { last_literal_offset_ = patch.LiteralOffset(); last_patch_offset_ = patch_offset; last_target_offset_ = target_offset; @@ -277,7 +277,7 @@ TEST_F(MultiOatRelativePatcherTest, Patch) { uint32_t method2_target_offset = 0xccccu; LinkerPatch method2_patch = LinkerPatch::DexCacheArrayPatch(method2_literal_offset, nullptr, 0u, 1234u); - patcher_.PatchDexCacheReference( + patcher_.PatchPcRelativeReference( &code, method2_patch, method2_patch_offset, method2_target_offset); DCHECK_EQ(method2_literal_offset, mock_->last_literal_offset_); DCHECK_EQ(method2_patch_offset + adjustment1, mock_->last_patch_offset_); diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc index 6727c17583..3a229831d0 100644 --- a/compiler/linker/relative_patcher.cc +++ b/compiler/linker/relative_patcher.cc @@ -62,10 +62,10 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create( LOG(FATAL) << "Unexpected relative call patch."; } - virtual void PatchDexCacheReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED, - const LinkerPatch& patch ATTRIBUTE_UNUSED, - uint32_t patch_offset ATTRIBUTE_UNUSED, - uint32_t target_offset ATTRIBUTE_UNUSED) { + void PatchPcRelativeReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED, + const LinkerPatch& patch ATTRIBUTE_UNUSED, + uint32_t patch_offset ATTRIBUTE_UNUSED, + uint32_t target_offset ATTRIBUTE_UNUSED) OVERRIDE { LOG(FATAL) << "Unexpected relative dex cache array patch."; } diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h index ba374512a1..a22b9f2c2d 100644 --- a/compiler/linker/relative_patcher.h +++ b/compiler/linker/relative_patcher.h @@ -104,10 +104,10 @@ class RelativePatcher { uint32_t target_offset) = 0; // Patch a reference to a dex cache location. - virtual void PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) = 0; + virtual void PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) = 0; protected: RelativePatcher() diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h index 704135a7b5..bf61ea0570 100644 --- a/compiler/linker/relative_patcher_test.h +++ b/compiler/linker/relative_patcher_test.h @@ -142,20 +142,27 @@ class RelativePatcherTest : public testing::Test { patched_code_.assign(code.begin(), code.end()); code = ArrayRef<const uint8_t>(patched_code_); for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.Type() == kLinkerPatchCallRelative) { + if (patch.GetType() == LinkerPatch::Type::kCallRelative) { auto result = method_offset_map_.FindMethodOffset(patch.TargetMethod()); uint32_t target_offset = result.first ? result.second : kTrampolineOffset + compiled_method->CodeDelta(); patcher_->PatchCall(&patched_code_, patch.LiteralOffset(), offset + patch.LiteralOffset(), target_offset); - } else if (patch.Type() == kLinkerPatchDexCacheArray) { + } else if (patch.GetType() == LinkerPatch::Type::kDexCacheArray) { uint32_t target_offset = dex_cache_arrays_begin_ + patch.TargetDexCacheElementOffset(); - patcher_->PatchDexCacheReference(&patched_code_, - patch, - offset + patch.LiteralOffset(), - target_offset); + patcher_->PatchPcRelativeReference(&patched_code_, + patch, + offset + patch.LiteralOffset(), + target_offset); + } else if (patch.GetType() == LinkerPatch::Type::kStringRelative) { + uint32_t target_offset = string_index_to_offset_map_.Get(patch.TargetStringIndex()); + patcher_->PatchPcRelativeReference(&patched_code_, + patch, + offset + patch.LiteralOffset(), + target_offset); } else { - LOG(FATAL) << "Bad patch type."; + LOG(FATAL) << "Bad patch type. " << patch.GetType(); + UNREACHABLE(); } } } @@ -257,6 +264,7 @@ class RelativePatcherTest : public testing::Test { MethodOffsetMap method_offset_map_; std::unique_ptr<RelativePatcher> patcher_; uint32_t dex_cache_arrays_begin_; + SafeMap<uint32_t, uint32_t> string_index_to_offset_map_; std::vector<MethodReference> compiled_method_refs_; std::vector<std::unique_ptr<CompiledMethod>> compiled_methods_; std::vector<uint8_t> patched_code_; diff --git a/compiler/linker/x86/relative_patcher_x86.cc b/compiler/linker/x86/relative_patcher_x86.cc index 24b1481c31..768d31abf4 100644 --- a/compiler/linker/x86/relative_patcher_x86.cc +++ b/compiler/linker/x86/relative_patcher_x86.cc @@ -21,10 +21,10 @@ namespace art { namespace linker { -void X86RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) { +void X86RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) { uint32_t anchor_literal_offset = patch.PcInsnOffset(); uint32_t literal_offset = patch.LiteralOffset(); diff --git a/compiler/linker/x86/relative_patcher_x86.h b/compiler/linker/x86/relative_patcher_x86.h index ddc244c269..fbf9ad4671 100644 --- a/compiler/linker/x86/relative_patcher_x86.h +++ b/compiler/linker/x86/relative_patcher_x86.h @@ -26,10 +26,10 @@ class X86RelativePatcher FINAL : public X86BaseRelativePatcher { public: X86RelativePatcher() { } - void PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) OVERRIDE; + void PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) OVERRIDE; }; } // namespace linker diff --git a/compiler/linker/x86/relative_patcher_x86_test.cc b/compiler/linker/x86/relative_patcher_x86_test.cc index 7acc33004a..2a44b7990e 100644 --- a/compiler/linker/x86/relative_patcher_x86_test.cc +++ b/compiler/linker/x86/relative_patcher_x86_test.cc @@ -70,15 +70,19 @@ TEST_F(X86RelativePatcherTest, CallOther) { uint32_t diff_after = method2_offset - (method1_offset + kCallCode.size() /* PC adjustment */); static const uint8_t method1_expected_code[] = { 0xe8, - static_cast<uint8_t>(diff_after), static_cast<uint8_t>(diff_after >> 8), - static_cast<uint8_t>(diff_after >> 16), static_cast<uint8_t>(diff_after >> 24) + static_cast<uint8_t>(diff_after), + static_cast<uint8_t>(diff_after >> 8), + static_cast<uint8_t>(diff_after >> 16), + static_cast<uint8_t>(diff_after >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code))); uint32_t diff_before = method1_offset - (method2_offset + kCallCode.size() /* PC adjustment */); static const uint8_t method2_expected_code[] = { 0xe8, - static_cast<uint8_t>(diff_before), static_cast<uint8_t>(diff_before >> 8), - static_cast<uint8_t>(diff_before >> 16), static_cast<uint8_t>(diff_before >> 24) + static_cast<uint8_t>(diff_before), + static_cast<uint8_t>(diff_before >> 8), + static_cast<uint8_t>(diff_before >> 16), + static_cast<uint8_t>(diff_before >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code))); } @@ -95,8 +99,10 @@ TEST_F(X86RelativePatcherTest, CallTrampoline) { uint32_t diff = kTrampolineOffset - (result.second + kCallCode.size()); static const uint8_t expected_code[] = { 0xe8, - static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), - static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24) + static_cast<uint8_t>(diff), + static_cast<uint8_t>(diff >> 8), + static_cast<uint8_t>(diff >> 16), + static_cast<uint8_t>(diff >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } @@ -125,8 +131,42 @@ TEST_F(X86RelativePatcherTest, DexCacheReference) { 0xe8, 0x00, 0x00, 0x00, 0x00, // call +0 0x5b, // pop ebx 0x8b, 0x83, // mov eax, [ebx + diff] - static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), - static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24) + static_cast<uint8_t>(diff), + static_cast<uint8_t>(diff >> 8), + static_cast<uint8_t>(diff >> 16), + static_cast<uint8_t>(diff >> 24) + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); +} + +TEST_F(X86RelativePatcherTest, StringReference) { + constexpr uint32_t kStringIndex = 1u; + constexpr uint32_t kStringOffset = 0x12345678; + string_index_to_offset_map_.Put(kStringIndex, kStringOffset); + static const uint8_t raw_code[] = { + 0xe8, 0x00, 0x00, 0x00, 0x00, // call +0 + 0x5b, // pop ebx + 0x8d, 0x83, 0x00, 0x01, 0x00, 0x00, // lea eax, [ebx + 256 (kDummy32BitValue)] + }; + constexpr uint32_t anchor_offset = 5u; // After call +0. + ArrayRef<const uint8_t> code(raw_code); + LinkerPatch patches[] = { + LinkerPatch::RelativeStringPatch(code.size() - 4u, nullptr, anchor_offset, kStringIndex), + }; + AddCompiledMethod(MethodRef(1u), code, ArrayRef<const LinkerPatch>(patches)); + Link(); + + auto result = method_offset_map_.FindMethodOffset(MethodRef(1u)); + ASSERT_TRUE(result.first); + uint32_t diff = kStringOffset - (result.second + anchor_offset); + static const uint8_t expected_code[] = { + 0xe8, 0x00, 0x00, 0x00, 0x00, // call +0 + 0x5b, // pop ebx + 0x8d, 0x83, // lea eax, [ebx + diff] + static_cast<uint8_t>(diff), + static_cast<uint8_t>(diff >> 8), + static_cast<uint8_t>(diff >> 16), + static_cast<uint8_t>(diff >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.cc b/compiler/linker/x86_64/relative_patcher_x86_64.cc index e571f50d2f..2ff69308c4 100644 --- a/compiler/linker/x86_64/relative_patcher_x86_64.cc +++ b/compiler/linker/x86_64/relative_patcher_x86_64.cc @@ -21,10 +21,10 @@ namespace art { namespace linker { -void X86_64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) { +void X86_64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) { DCHECK_LE(patch.LiteralOffset() + 4u, code->size()); // Unsigned arithmetic with its well-defined overflow behavior is just fine here. uint32_t displacement = target_offset - patch_offset; diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.h b/compiler/linker/x86_64/relative_patcher_x86_64.h index feecb3a2ad..11bb6d59e3 100644 --- a/compiler/linker/x86_64/relative_patcher_x86_64.h +++ b/compiler/linker/x86_64/relative_patcher_x86_64.h @@ -26,10 +26,10 @@ class X86_64RelativePatcher FINAL : public X86BaseRelativePatcher { public: X86_64RelativePatcher() { } - void PatchDexCacheReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) OVERRIDE; + void PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) OVERRIDE; }; } // namespace linker diff --git a/compiler/linker/x86_64/relative_patcher_x86_64_test.cc b/compiler/linker/x86_64/relative_patcher_x86_64_test.cc index 36e0f01a50..2b46453255 100644 --- a/compiler/linker/x86_64/relative_patcher_x86_64_test.cc +++ b/compiler/linker/x86_64/relative_patcher_x86_64_test.cc @@ -29,6 +29,8 @@ class X86_64RelativePatcherTest : public RelativePatcherTest { static const ArrayRef<const uint8_t> kCallCode; static const uint8_t kDexCacheLoadRawCode[]; static const ArrayRef<const uint8_t> kDexCacheLoadCode; + static const uint8_t kStringReferenceRawCode[]; + static const ArrayRef<const uint8_t> kStringReferenceCode; uint32_t GetMethodOffset(uint32_t method_idx) { auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx)); @@ -51,6 +53,14 @@ const uint8_t X86_64RelativePatcherTest::kDexCacheLoadRawCode[] = { const ArrayRef<const uint8_t> X86_64RelativePatcherTest::kDexCacheLoadCode( kDexCacheLoadRawCode); +const uint8_t X86_64RelativePatcherTest::kStringReferenceRawCode[] = { + 0x8d, 0x05, // lea eax, [rip + <offset>] + 0x00, 0x01, 0x00, 0x00 +}; + +const ArrayRef<const uint8_t> X86_64RelativePatcherTest::kStringReferenceCode( + kStringReferenceRawCode); + TEST_F(X86_64RelativePatcherTest, CallSelf) { LinkerPatch patches[] = { LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 1u), @@ -80,15 +90,19 @@ TEST_F(X86_64RelativePatcherTest, CallOther) { uint32_t diff_after = method2_offset - (method1_offset + kCallCode.size() /* PC adjustment */); static const uint8_t method1_expected_code[] = { 0xe8, - static_cast<uint8_t>(diff_after), static_cast<uint8_t>(diff_after >> 8), - static_cast<uint8_t>(diff_after >> 16), static_cast<uint8_t>(diff_after >> 24) + static_cast<uint8_t>(diff_after), + static_cast<uint8_t>(diff_after >> 8), + static_cast<uint8_t>(diff_after >> 16), + static_cast<uint8_t>(diff_after >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code))); uint32_t diff_before = method1_offset - (method2_offset + kCallCode.size() /* PC adjustment */); static const uint8_t method2_expected_code[] = { 0xe8, - static_cast<uint8_t>(diff_before), static_cast<uint8_t>(diff_before >> 8), - static_cast<uint8_t>(diff_before >> 16), static_cast<uint8_t>(diff_before >> 24) + static_cast<uint8_t>(diff_before), + static_cast<uint8_t>(diff_before >> 8), + static_cast<uint8_t>(diff_before >> 16), + static_cast<uint8_t>(diff_before >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code))); } @@ -105,8 +119,10 @@ TEST_F(X86_64RelativePatcherTest, CallTrampoline) { uint32_t diff = kTrampolineOffset - (result.second + kCallCode.size()); static const uint8_t expected_code[] = { 0xe8, - static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), - static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24) + static_cast<uint8_t>(diff), + static_cast<uint8_t>(diff >> 8), + static_cast<uint8_t>(diff >> 16), + static_cast<uint8_t>(diff >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } @@ -126,8 +142,34 @@ TEST_F(X86_64RelativePatcherTest, DexCacheReference) { dex_cache_arrays_begin_ + kElementOffset - (result.second + kDexCacheLoadCode.size()); static const uint8_t expected_code[] = { 0x8b, 0x05, - static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8), - static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24) + static_cast<uint8_t>(diff), + static_cast<uint8_t>(diff >> 8), + static_cast<uint8_t>(diff >> 16), + static_cast<uint8_t>(diff >> 24) + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); +} + +TEST_F(X86_64RelativePatcherTest, StringReference) { + constexpr uint32_t kStringIndex = 1u; + constexpr uint32_t kStringOffset = 0x12345678; + string_index_to_offset_map_.Put(kStringIndex, kStringOffset); + LinkerPatch patches[] = { + LinkerPatch::RelativeStringPatch( + kStringReferenceCode.size() - 4u, nullptr, 0u, kStringIndex), + }; + AddCompiledMethod(MethodRef(1u), kStringReferenceCode, ArrayRef<const LinkerPatch>(patches)); + Link(); + + auto result = method_offset_map_.FindMethodOffset(MethodRef(1u)); + ASSERT_TRUE(result.first); + uint32_t diff = kStringOffset - (result.second + kStringReferenceCode.size()); + static const uint8_t expected_code[] = { + 0x8d, 0x05, + static_cast<uint8_t>(diff), + static_cast<uint8_t>(diff >> 8), + static_cast<uint8_t>(diff >> 16), + static_cast<uint8_t>(diff >> 24) }; EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code))); } diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index c2f19c9d61..3a67b1ec2a 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -1046,6 +1046,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { OatDexMethodVisitor::StartClass(dex_file, class_def_index); if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) { dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file); + DCHECK(dex_cache_ != nullptr); } return true; } @@ -1115,28 +1116,56 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { quick_code = ArrayRef<const uint8_t>(patched_code_); for (const LinkerPatch& patch : compiled_method->GetPatches()) { uint32_t literal_offset = patch.LiteralOffset(); - if (patch.Type() == kLinkerPatchCallRelative) { - // NOTE: Relative calls across oat files are not supported. - uint32_t target_offset = GetTargetOffset(patch); - writer_->relative_patcher_->PatchCall(&patched_code_, - literal_offset, - offset_ + literal_offset, - target_offset); - } else if (patch.Type() == kLinkerPatchDexCacheArray) { - uint32_t target_offset = GetDexCacheOffset(patch); - writer_->relative_patcher_->PatchDexCacheReference(&patched_code_, - patch, - offset_ + literal_offset, - target_offset); - } else if (patch.Type() == kLinkerPatchCall) { - uint32_t target_offset = GetTargetOffset(patch); - PatchCodeAddress(&patched_code_, literal_offset, target_offset); - } else if (patch.Type() == kLinkerPatchMethod) { - ArtMethod* method = GetTargetMethod(patch); - PatchMethodAddress(&patched_code_, literal_offset, method); - } else if (patch.Type() == kLinkerPatchType) { - mirror::Class* type = GetTargetType(patch); - PatchObjectAddress(&patched_code_, literal_offset, type); + switch (patch.GetType()) { + case LinkerPatch::Type::kCallRelative: { + // NOTE: Relative calls across oat files are not supported. + uint32_t target_offset = GetTargetOffset(patch); + writer_->relative_patcher_->PatchCall(&patched_code_, + literal_offset, + offset_ + literal_offset, + target_offset); + break; + } + case LinkerPatch::Type::kDexCacheArray: { + uint32_t target_offset = GetDexCacheOffset(patch); + writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, + patch, + offset_ + literal_offset, + target_offset); + break; + } + case LinkerPatch::Type::kStringRelative: { + uint32_t target_offset = GetTargetObjectOffset(GetTargetString(patch)); + writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, + patch, + offset_ + literal_offset, + target_offset); + break; + } + case LinkerPatch::Type::kCall: { + uint32_t target_offset = GetTargetOffset(patch); + PatchCodeAddress(&patched_code_, literal_offset, target_offset); + break; + } + case LinkerPatch::Type::kMethod: { + ArtMethod* method = GetTargetMethod(patch); + PatchMethodAddress(&patched_code_, literal_offset, method); + break; + } + case LinkerPatch::Type::kString: { + mirror::String* string = GetTargetString(patch); + PatchObjectAddress(&patched_code_, literal_offset, string); + break; + } + case LinkerPatch::Type::kType: { + mirror::Class* type = GetTargetType(patch); + PatchObjectAddress(&patched_code_, literal_offset, type); + break; + } + default: { + DCHECK_EQ(patch.GetType(), LinkerPatch::Type::kRecordPosition); + break; + } } } } @@ -1205,15 +1234,23 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { return target_offset; } - mirror::Class* GetTargetType(const LinkerPatch& patch) - SHARED_REQUIRES(Locks::mutator_lock_) { + mirror::Class* GetTargetType(const LinkerPatch& patch) SHARED_REQUIRES(Locks::mutator_lock_) { mirror::DexCache* dex_cache = (dex_file_ == patch.TargetTypeDexFile()) - ? dex_cache_ : class_linker_->FindDexCache(Thread::Current(), *patch.TargetTypeDexFile()); + ? dex_cache_ + : class_linker_->FindDexCache(Thread::Current(), *patch.TargetTypeDexFile()); mirror::Class* type = dex_cache->GetResolvedType(patch.TargetTypeIndex()); CHECK(type != nullptr); return type; } + mirror::String* GetTargetString(const LinkerPatch& patch) SHARED_REQUIRES(Locks::mutator_lock_) { + mirror::String* string = dex_cache_->GetResolvedString(patch.TargetStringIndex()); + DCHECK(string != nullptr); + DCHECK(writer_->HasBootImage() || + Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(string)); + return string; + } + uint32_t GetDexCacheOffset(const LinkerPatch& patch) SHARED_REQUIRES(Locks::mutator_lock_) { if (writer_->HasBootImage()) { uintptr_t element = writer_->image_writer_->GetDexCacheArrayElementImageAddress<uintptr_t>( @@ -1227,6 +1264,15 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } } + uint32_t GetTargetObjectOffset(mirror::Object* object) SHARED_REQUIRES(Locks::mutator_lock_) { + DCHECK(writer_->HasBootImage()); + object = writer_->image_writer_->GetImageAddress(object); + size_t oat_index = writer_->image_writer_->GetOatIndexForDexFile(dex_file_); + uintptr_t oat_data_begin = writer_->image_writer_->GetOatDataBegin(oat_index); + // TODO: Clean up offset types. The target offset must be treated as signed. + return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(object) - oat_data_begin); + } + void PatchObjectAddress(std::vector<uint8_t>* code, uint32_t offset, mirror::Object* object) SHARED_REQUIRES(Locks::mutator_lock_) { if (writer_->HasBootImage()) { diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc index 082d15961a..b6b8322f03 100644 --- a/compiler/optimizing/builder.cc +++ b/compiler/optimizing/builder.cc @@ -106,7 +106,6 @@ void HGraphBuilder::If_22t(const Instruction& instruction, uint32_t dex_pc) { HBasicBlock* fallthrough_target = FindBlockStartingAt(dex_pc + instruction.SizeInCodeUnits()); DCHECK(branch_target != nullptr); DCHECK(fallthrough_target != nullptr); - PotentiallyAddSuspendCheck(branch_target, dex_pc); HInstruction* first = LoadLocal(instruction.VRegA(), Primitive::kPrimInt, dex_pc); HInstruction* second = LoadLocal(instruction.VRegB(), Primitive::kPrimInt, dex_pc); T* comparison = new (arena_) T(first, second, dex_pc); @@ -125,7 +124,6 @@ void HGraphBuilder::If_21t(const Instruction& instruction, uint32_t dex_pc) { HBasicBlock* fallthrough_target = FindBlockStartingAt(dex_pc + instruction.SizeInCodeUnits()); DCHECK(branch_target != nullptr); DCHECK(fallthrough_target != nullptr); - PotentiallyAddSuspendCheck(branch_target, dex_pc); HInstruction* value = LoadLocal(instruction.VRegA(), Primitive::kPrimInt, dex_pc); T* comparison = new (arena_) T(value, graph_->GetIntConstant(0, dex_pc), dex_pc); current_block_->AddInstruction(comparison); @@ -1788,7 +1786,6 @@ void HGraphBuilder::BuildSwitchCaseHelper(const Instruction& instruction, size_t int32_t target_offset, uint32_t dex_pc) { HBasicBlock* case_target = FindBlockStartingAt(dex_pc + target_offset); DCHECK(case_target != nullptr); - PotentiallyAddSuspendCheck(case_target, dex_pc); // The current case's value. HInstruction* this_case_value = graph_->GetIntConstant(case_value_int, dex_pc); @@ -1824,23 +1821,6 @@ void HGraphBuilder::BuildSwitchCaseHelper(const Instruction& instruction, size_t } } -void HGraphBuilder::PotentiallyAddSuspendCheck(HBasicBlock* target, uint32_t dex_pc) { - int32_t target_offset = target->GetDexPc() - dex_pc; - if (target_offset <= 0) { - // DX generates back edges to the first encountered return. We can save - // time of later passes by not adding redundant suspend checks. - HInstruction* last_in_target = target->GetLastInstruction(); - if (last_in_target != nullptr && - (last_in_target->IsReturn() || last_in_target->IsReturnVoid())) { - return; - } - - // Add a suspend check to backward branches which may potentially loop. We - // can remove them after we recognize loops in the graph. - current_block_->AddInstruction(new (arena_) HSuspendCheck(dex_pc)); - } -} - bool HGraphBuilder::CanDecodeQuickenedInfo() const { return interpreter_metadata_ != nullptr; } @@ -1972,7 +1952,6 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 int32_t offset = instruction.GetTargetOffset(); HBasicBlock* target = FindBlockStartingAt(offset + dex_pc); DCHECK(target != nullptr); - PotentiallyAddSuspendCheck(target, dex_pc); current_block_->AddInstruction(new (arena_) HGoto(dex_pc)); current_block_->AddSuccessor(target); current_block_ = nullptr; @@ -2766,20 +2745,16 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 case Instruction::CONST_STRING: { uint32_t string_index = instruction.VRegB_21c(); - bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache( - *dex_file_, string_index); current_block_->AddInstruction( - new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache)); + new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc)); UpdateLocal(instruction.VRegA_21c(), current_block_->GetLastInstruction(), dex_pc); break; } case Instruction::CONST_STRING_JUMBO: { uint32_t string_index = instruction.VRegB_31c(); - bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache( - *dex_file_, string_index); current_block_->AddInstruction( - new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache)); + new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc)); UpdateLocal(instruction.VRegA_31c(), current_block_->GetLastInstruction(), dex_pc); break; } diff --git a/compiler/optimizing/builder.h b/compiler/optimizing/builder.h index e3dd0e8216..48f5316222 100644 --- a/compiler/optimizing/builder.h +++ b/compiler/optimizing/builder.h @@ -133,7 +133,6 @@ class HGraphBuilder : public ValueObject { HLocal* GetLocalAt(uint32_t register_index) const; void UpdateLocal(uint32_t register_index, HInstruction* instruction, uint32_t dex_pc) const; HInstruction* LoadLocal(uint32_t register_index, Primitive::Type type, uint32_t dex_pc) const; - void PotentiallyAddSuspendCheck(HBasicBlock* target, uint32_t dex_pc); void InitializeParameters(uint16_t number_of_parameters); // Returns whether the current method needs access check for the type. diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index 32869ec0b4..f1139218be 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -54,7 +54,6 @@ #include "parallel_move_resolver.h" #include "ssa_liveness_analysis.h" #include "utils/assembler.h" -#include "verifier/dex_gc_map.h" #include "vmap_table.h" namespace art { diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index e56323ff0f..cad55296bc 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -443,6 +443,11 @@ class CodeGenerator { uint32_t dex_pc, SlowPathCode* slow_path) = 0; + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back info that should be used instead. + virtual HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) = 0; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. virtual HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -471,6 +476,18 @@ class CodeGenerator { LabelType label; }; + // String patch info used for recording locations of required linker patches and + // target strings. The actual string address can be absolute or PC-relative. + template <typename LabelType> + struct StringPatchInfo { + StringPatchInfo(const DexFile& df, uint32_t index) + : dex_file(df), string_index(index), label() { } + + const DexFile& dex_file; + uint32_t string_index; + LabelType label; + }; + CodeGenerator(HGraph* graph, size_t number_of_core_registers, size_t number_of_fpu_registers, diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 4c9c25fbd5..98577d67ea 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -779,13 +779,19 @@ CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, move_resolver_(graph->GetArena(), this), assembler_(), isa_features_(isa_features), + uint32_literals_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), method_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), call_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - dex_cache_arrays_base_labels_(std::less<HArmDexCacheArraysBase*>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_address_patches_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Always save the LR register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(LR)); } @@ -1412,13 +1418,13 @@ void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instructio // Nothing to do. The code always falls through. return; } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against 1. - if (cond->AsIntConstant()->IsOne()) { + // Constant condition, statically compared against "true" (integer value 1). + if (cond->AsIntConstant()->IsTrue()) { if (true_target != nullptr) { __ b(true_target); } } else { - DCHECK(cond->AsIntConstant()->IsZero()); + DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); if (false_target != nullptr) { __ b(false_target); } @@ -3221,7 +3227,7 @@ void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { if (rhs.IsConstant()) { uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); // Map all rotations to +ve. equivalents on the interval [0,63]. - rot &= kMaxLongShiftValue; + rot &= kMaxLongShiftDistance; // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate // logic below to a simple pair of binary orr. // (e.g. 34 bits == in_reg swap + 2 bits right.) @@ -3374,7 +3380,7 @@ void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { if (second.IsRegister()) { Register second_reg = second.AsRegister<Register>(); // ARM doesn't mask the shift count so we need to do it ourselves. - __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftValue)); + __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); if (op->IsShl()) { __ Lsl(out_reg, first_reg, out_reg); } else if (op->IsShr()) { @@ -3384,7 +3390,7 @@ void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { } } else { int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t shift_value = static_cast<uint32_t>(cst & kMaxIntShiftValue); + uint32_t shift_value = cst & kMaxIntShiftDistance; if (shift_value == 0) { // ARM does not support shifting with 0 immediate. __ Mov(out_reg, first_reg); } else if (op->IsShl()) { @@ -3410,7 +3416,7 @@ void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { Register second_reg = second.AsRegister<Register>(); if (op->IsShl()) { - __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftValue)); + __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); // Shift the high part __ Lsl(o_h, high, o_l); // Shift the low part and `or` what overflew on the high part @@ -3424,7 +3430,7 @@ void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { // Shift the low part __ Lsl(o_l, low, o_l); } else if (op->IsShr()) { - __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftValue)); + __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); // Shift the low part __ Lsr(o_l, low, o_h); // Shift the high part and `or` what underflew on the low part @@ -3438,7 +3444,7 @@ void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { // Shift the high part __ Asr(o_h, high, o_h); } else { - __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftValue)); + __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); // same as Shr except we use `Lsr`s and not `Asr`s __ Lsr(o_l, low, o_h); __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); @@ -3454,7 +3460,7 @@ void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { DCHECK_NE(o_l, high); DCHECK_NE(o_h, low); int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t shift_value = static_cast<uint32_t>(cst & kMaxLongShiftValue); + uint32_t shift_value = cst & kMaxLongShiftDistance; if (shift_value > 32) { if (op->IsShl()) { __ Lsl(o_h, low, shift_value - 32); @@ -5221,12 +5227,57 @@ void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( __ Bind(slow_path->GetExitLabel()); } +HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); + // We disable pc-relative load when there is an irreducible loop, as the optimization + // is incompatible with it. + // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods + // with irreducible loops. + if (GetGraph()->HasIrreducibleLoops()) { + return HLoadString::LoadKind::kDexCacheViaMethod; + } + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderARM::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + HLoadString::LoadKind load_kind = load->GetLoadKind(); + if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod || + load_kind == HLoadString::LoadKind::kDexCachePcRelative) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } @@ -5234,16 +5285,73 @@ void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); - Register current_method = locations->InAt(0).AsRegister<Register>(); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value()); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: { + DCHECK(!kEmitCompilerReadBarrier); + __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + CodeGeneratorARM::PcRelativePatchInfo* labels = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); + __ BindTrackedLabel(&labels->movw_label); + __ movw(out, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->movt_label); + __ movt(out, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->add_pc_label); + __ add(out, out, ShifterOperand(PC)); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives + // a 128B range. To try and reduce the number of literals if we load multiple strings, + // simply split the dex cache address to a 128B aligned base loaded from a literal + // and the remaining offset embedded in the load. + static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes."); + DCHECK_ALIGNED(load->GetAddress(), 4u); + constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; + uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); + uint32_t offset = address & MaxInt<uint32_t>(offset_bits); + __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); + GenerateGcRootFieldLoad(load, out_loc, out, offset); + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + Register base_reg = locations->InAt(0).AsRegister<Register>(); + HArmDexCacheArraysBase* base = load->InputAt(0)->AsArmDexCacheArraysBase(); + int32_t offset = load->GetDexCacheElementOffset() - base->GetElementOffset(); + GenerateGcRootFieldLoad(load, out_loc, base_reg, offset); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + Register current_method = locations->InAt(0).AsRegister<Register>(); + + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value()); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); @@ -6220,6 +6328,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOr HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; // We disable pc-relative load when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods + // with irreducible loops. if (GetGraph()->HasIrreducibleLoops() && (dispatch_info.method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { @@ -6399,13 +6509,49 @@ void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp __ blx(LR); } +CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( + const DexFile& dex_file, uint32_t string_index) { + return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); +} + +CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( + const DexFile& dex_file, uint32_t element_offset) { + return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); +} + +CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( + const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { + patches->emplace_back(dex_file, offset_or_index); + return &patches->back(); +} + +Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, + uint32_t string_index) { + return boot_image_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { + bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); + Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); +} + +Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { + return DeduplicateUint32Literal(address, &uint32_literals_); +} + void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + call_patches_.size() + relative_call_patches_.size() + - /* MOVW+MOVT for each base */ 2u * dex_cache_arrays_base_labels_.size(); + /* MOVW+MOVT for each base */ 2u * pc_relative_dex_cache_patches_.size() + + boot_image_string_patches_.size() + + /* MOVW+MOVT for each base */ 2u * pc_relative_string_patches_.size() + + boot_image_address_patches_.size(); linker_patches->reserve(size); for (const auto& entry : method_patches_) { const MethodReference& target_method = entry.first; @@ -6431,41 +6577,75 @@ void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche info.target_method.dex_file, info.target_method.dex_method_index)); } - for (const auto& pair : dex_cache_arrays_base_labels_) { - HArmDexCacheArraysBase* base = pair.first; - const DexCacheArraysBaseLabels* labels = &pair.second; - const DexFile& dex_file = base->GetDexFile(); - size_t base_element_offset = base->GetElementOffset(); - DCHECK(labels->add_pc_label.IsBound()); - uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(labels->add_pc_label.Position()); + for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { + const DexFile& dex_file = info.target_dex_file; + size_t base_element_offset = info.offset_or_index; + DCHECK(info.add_pc_label.IsBound()); + uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); // Add MOVW patch. - DCHECK(labels->movw_label.IsBound()); - uint32_t movw_offset = dchecked_integral_cast<uint32_t>(labels->movw_label.Position()); + DCHECK(info.movw_label.IsBound()); + uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movw_offset, &dex_file, add_pc_offset, base_element_offset)); // Add MOVT patch. - DCHECK(labels->movt_label.IsBound()); - uint32_t movt_offset = dchecked_integral_cast<uint32_t>(labels->movt_label.Position()); + DCHECK(info.movt_label.IsBound()); + uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movt_offset, &dex_file, add_pc_offset, base_element_offset)); } + for (const auto& entry : boot_image_string_patches_) { + const StringReference& target_string = entry.first; + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = literal->GetLabel()->Position(); + linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, + target_string.dex_file, + target_string.string_index)); + } + for (const PcRelativePatchInfo& info : pc_relative_string_patches_) { + const DexFile& dex_file = info.target_dex_file; + uint32_t string_index = info.offset_or_index; + DCHECK(info.add_pc_label.IsBound()); + uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); + // Add MOVW patch. + DCHECK(info.movw_label.IsBound()); + uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); + linker_patches->push_back(LinkerPatch::RelativeStringPatch(movw_offset, + &dex_file, + add_pc_offset, + string_index)); + // Add MOVT patch. + DCHECK(info.movt_label.IsBound()); + uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); + linker_patches->push_back(LinkerPatch::RelativeStringPatch(movt_offset, + &dex_file, + add_pc_offset, + string_index)); + } + for (const auto& entry : boot_image_address_patches_) { + DCHECK(GetCompilerOptions().GetIncludePatchInformation()); + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = literal->GetLabel()->Position(); + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } +} + +Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { + return map->GetOrCreate( + value, + [this, value]() { return __ NewLiteral<uint32_t>(value); }); } Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map) { - // Look up the literal for target_method. - auto lb = map->lower_bound(target_method); - if (lb != map->end() && !map->key_comp()(target_method, lb->first)) { - return lb->second; - } - // We don't have a literal for this method yet, insert a new one. - Literal* literal = __ NewLiteral<uint32_t>(0u); - map->PutBefore(lb, target_method, literal); - return literal; + return map->GetOrCreate( + target_method, + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); } Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) { @@ -6600,16 +6780,16 @@ void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); locations->SetOut(Location::RequiresRegister()); - codegen_->AddDexCacheArraysBase(base); } void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); - CodeGeneratorARM::DexCacheArraysBaseLabels* labels = codegen_->GetDexCacheArraysBaseLabels(base); + CodeGeneratorARM::PcRelativePatchInfo* labels = + codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); __ BindTrackedLabel(&labels->movw_label); - __ movw(base_reg, 0u); + __ movw(base_reg, /* placeholder */ 0u); __ BindTrackedLabel(&labels->movt_label); - __ movt(base_reg, 0u); + __ movt(base_reg, /* placeholder */ 0u); __ BindTrackedLabel(&labels->add_pc_label); __ add(base_reg, base_reg, ShifterOperand(PC)); } diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index cc4aa144c0..84341284a3 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -23,6 +23,7 @@ #include "nodes.h" #include "parallel_move_resolver.h" #include "utils/arm/assembler_thumb2.h" +#include "utils/string_reference.h" namespace art { namespace arm { @@ -403,6 +404,11 @@ class CodeGeneratorARM : public CodeGenerator { Label* GetFrameEntryLabel() { return &frame_entry_label_; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -414,32 +420,34 @@ class CodeGeneratorARM : public CodeGenerator { void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; - void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; - - // The PC-relative base address is loaded with three instructions, MOVW+MOVT + // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays + // and boot image strings. The only difference is the interpretation of the offset_or_index. + // The PC-relative address is loaded with three instructions, MOVW+MOVT // to load the offset to base_reg and then ADD base_reg, PC. The offset is // calculated from the ADD's effective PC, i.e. PC+4 on Thumb2. Though we // currently emit these 3 instructions together, instruction scheduling could // split this sequence apart, so we keep separate labels for each of them. - struct DexCacheArraysBaseLabels { - DexCacheArraysBaseLabels() = default; - DexCacheArraysBaseLabels(DexCacheArraysBaseLabels&& other) = default; - + struct PcRelativePatchInfo { + PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) + : target_dex_file(dex_file), offset_or_index(off_or_idx) { } + PcRelativePatchInfo(PcRelativePatchInfo&& other) = default; + + const DexFile& target_dex_file; + // Either the dex cache array element offset or the string index. + uint32_t offset_or_index; Label movw_label; Label movt_label; Label add_pc_label; }; - void AddDexCacheArraysBase(HArmDexCacheArraysBase* base) { - DexCacheArraysBaseLabels labels; - dex_cache_arrays_base_labels_.Put(base, std::move(labels)); - } + PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, uint32_t string_index); + PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset); + Literal* DeduplicateBootImageStringLiteral(const DexFile& dex_file, uint32_t string_index); + Literal* DeduplicateBootImageAddressLiteral(uint32_t address); + Literal* DeduplicateDexCacheAddressLiteral(uint32_t address); - DexCacheArraysBaseLabels* GetDexCacheArraysBaseLabels(HArmDexCacheArraysBase* base) { - auto it = dex_cache_arrays_base_labels_.find(base); - DCHECK(it != dex_cache_arrays_base_labels_.end()); - return &it->second; - } + void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. @@ -525,14 +533,19 @@ class CodeGeneratorARM : public CodeGenerator { Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp); + using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>; - using DexCacheArraysBaseToLabelsMap = ArenaSafeMap<HArmDexCacheArraysBase*, - DexCacheArraysBaseLabels, - std::less<HArmDexCacheArraysBase*>>; + using BootStringToLiteralMap = ArenaSafeMap<StringReference, + Literal*, + StringReferenceValueComparator>; + Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); Literal* DeduplicateMethodAddressLiteral(MethodReference target_method); Literal* DeduplicateMethodCodeLiteral(MethodReference target_method); + PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, + uint32_t offset_or_index, + ArenaDeque<PcRelativePatchInfo>* patches); // Labels for each block that will be compiled. Label* block_labels_; // Indexed by block id. @@ -543,14 +556,22 @@ class CodeGeneratorARM : public CodeGenerator { Thumb2Assembler assembler_; const ArmInstructionSetFeatures& isa_features_; + // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. + Uint32ToLiteralMap uint32_literals_; // Method patch info, map MethodReference to a literal for method address and method code. MethodToLiteralMap method_patches_; MethodToLiteralMap call_patches_; // Relative call patch info. // Using ArenaDeque<> which retains element addresses on push/emplace_back(). ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_; - - DexCacheArraysBaseToLabelsMap dex_cache_arrays_base_labels_; + // PC-relative patch info for each HArmDexCacheArraysBase. + ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; + // Deduplication map for boot string literals for kBootImageLinkTimeAddress. + BootStringToLiteralMap boot_image_string_patches_; + // PC-relative String patch info. + ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; + // Deduplication map for patchable boot image addresses. + Uint32ToLiteralMap boot_image_address_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM); }; diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 088dbb3693..491014d3e5 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -905,6 +905,8 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, instruction_visitor_(graph, this), move_resolver_(graph->GetArena(), this), isa_features_(isa_features), + uint32_literals_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), method_patches_(MethodReferenceComparator(), @@ -912,7 +914,12 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, call_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_address_patches_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save the link register (containing the return address) to mimic Quick. AddAllocatedRegister(LocationFrom(lr)); } @@ -1818,9 +1825,8 @@ void InstructionCodeGeneratorARM64::HandleShift(HBinaryOperation* instr) { Register lhs = InputRegisterAt(instr, 0); Operand rhs = InputOperandAt(instr, 1); if (rhs.IsImmediate()) { - uint32_t shift_value = (type == Primitive::kPrimInt) - ? static_cast<uint32_t>(rhs.immediate() & kMaxIntShiftValue) - : static_cast<uint32_t>(rhs.immediate() & kMaxLongShiftValue); + uint32_t shift_value = rhs.immediate() & + (type == Primitive::kPrimInt ? kMaxIntShiftDistance : kMaxLongShiftDistance); if (instr->IsShl()) { __ Lsl(dst, lhs, shift_value); } else if (instr->IsShr()) { @@ -1921,9 +1927,8 @@ void InstructionCodeGeneratorARM64::VisitArm64DataProcWithShifterOp( // conversion) can have a different type from the current instruction's type, // so we manually indicate the type. Register right_reg = RegisterFrom(instruction->GetLocations()->InAt(1), type); - int64_t shift_amount = (type == Primitive::kPrimInt) - ? static_cast<uint32_t>(instruction->GetShiftAmount() & kMaxIntShiftValue) - : static_cast<uint32_t>(instruction->GetShiftAmount() & kMaxLongShiftValue); + int64_t shift_amount = instruction->GetShiftAmount() & + (type == Primitive::kPrimInt ? kMaxIntShiftDistance : kMaxLongShiftDistance); Operand right_operand(0); @@ -1949,7 +1954,7 @@ void InstructionCodeGeneratorARM64::VisitArm64DataProcWithShifterOp( __ And(out, left, right_operand); break; case HInstruction::kNeg: - DCHECK(instruction->InputAt(0)->AsConstant()->IsZero()); + DCHECK(instruction->InputAt(0)->AsConstant()->IsArithmeticZero()); __ Neg(out, right_operand); break; case HInstruction::kOr: @@ -1994,7 +1999,7 @@ void LocationsBuilderARM64::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) HInstruction* accumulator = instr->InputAt(HMultiplyAccumulate::kInputAccumulatorIndex); if (instr->GetOpKind() == HInstruction::kSub && accumulator->IsConstant() && - accumulator->AsConstant()->IsZero()) { + accumulator->AsConstant()->IsArithmeticZero()) { // Don't allocate register for Mneg instruction. } else { locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, @@ -2033,7 +2038,7 @@ void InstructionCodeGeneratorARM64::VisitMultiplyAccumulate(HMultiplyAccumulate* } else { DCHECK(instr->GetOpKind() == HInstruction::kSub); HInstruction* accum_instr = instr->InputAt(HMultiplyAccumulate::kInputAccumulatorIndex); - if (accum_instr->IsConstant() && accum_instr->AsConstant()->IsZero()) { + if (accum_instr->IsConstant() && accum_instr->AsConstant()->IsArithmeticZero()) { __ Mneg(res, mul_left, mul_right); } else { Register accumulator = InputRegisterAt(instr, HMultiplyAccumulate::kInputAccumulatorIndex); @@ -2378,9 +2383,35 @@ void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) { GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0)); } -static bool IsFloatingPointZeroConstant(HInstruction* instruction) { - return (instruction->IsFloatConstant() && (instruction->AsFloatConstant()->GetValue() == 0.0f)) - || (instruction->IsDoubleConstant() && (instruction->AsDoubleConstant()->GetValue() == 0.0)); +static bool IsFloatingPointZeroConstant(HInstruction* inst) { + return (inst->IsFloatConstant() && (inst->AsFloatConstant()->IsArithmeticZero())) + || (inst->IsDoubleConstant() && (inst->AsDoubleConstant()->IsArithmeticZero())); +} + +void InstructionCodeGeneratorARM64::GenerateFcmp(HInstruction* instruction) { + FPRegister lhs_reg = InputFPRegisterAt(instruction, 0); + Location rhs_loc = instruction->GetLocations()->InAt(1); + if (rhs_loc.IsConstant()) { + // 0.0 is the only immediate that can be encoded directly in + // an FCMP instruction. + // + // Both the JLS (section 15.20.1) and the JVMS (section 6.5) + // specify that in a floating-point comparison, positive zero + // and negative zero are considered equal, so we can use the + // literal 0.0 for both cases here. + // + // Note however that some methods (Float.equal, Float.compare, + // Float.compareTo, Double.equal, Double.compare, + // Double.compareTo, Math.max, Math.min, StrictMath.max, + // StrictMath.min) consider 0.0 to be (strictly) greater than + // -0.0. So if we ever translate calls to these methods into a + // HCompare instruction, we must handle the -0.0 case with + // care here. + DCHECK(IsFloatingPointZeroConstant(rhs_loc.GetConstant())); + __ Fcmp(lhs_reg, 0.0); + } else { + __ Fcmp(lhs_reg, InputFPRegisterAt(instruction, 1)); + } } void LocationsBuilderARM64::VisitCompare(HCompare* compare) { @@ -2438,14 +2469,7 @@ void InstructionCodeGeneratorARM64::VisitCompare(HCompare* compare) { case Primitive::kPrimFloat: case Primitive::kPrimDouble: { Register result = OutputRegister(compare); - FPRegister left = InputFPRegisterAt(compare, 0); - if (compare->GetLocations()->InAt(1).IsConstant()) { - DCHECK(IsFloatingPointZeroConstant(compare->GetLocations()->InAt(1).GetConstant())); - // 0.0 is the only immediate that can be encoded directly in an FCMP instruction. - __ Fcmp(left, 0.0); - } else { - __ Fcmp(left, InputFPRegisterAt(compare, 1)); - } + GenerateFcmp(compare); __ Cset(result, ne); __ Cneg(result, result, ARM64FPCondition(kCondLT, compare->IsGtBias())); break; @@ -2485,14 +2509,7 @@ void InstructionCodeGeneratorARM64::HandleCondition(HCondition* instruction) { IfCondition if_cond = instruction->GetCondition(); if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) { - FPRegister lhs = InputFPRegisterAt(instruction, 0); - if (locations->InAt(1).IsConstant()) { - DCHECK(IsFloatingPointZeroConstant(locations->InAt(1).GetConstant())); - // 0.0 is the only immediate that can be encoded directly in an FCMP instruction. - __ Fcmp(lhs, 0.0); - } else { - __ Fcmp(lhs, InputFPRegisterAt(instruction, 1)); - } + GenerateFcmp(instruction); __ Cset(res, ARM64FPCondition(if_cond, instruction->IsGtBias())); } else { // Integer cases. @@ -2823,13 +2840,13 @@ void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruct // Nothing to do. The code always falls through. return; } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against 1. - if (cond->AsIntConstant()->IsOne()) { + // Constant condition, statically compared against "true" (integer value 1). + if (cond->AsIntConstant()->IsTrue()) { if (true_target != nullptr) { __ B(true_target); } } else { - DCHECK(cond->AsIntConstant()->IsZero()); + DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); if (false_target != nullptr) { __ B(false_target); } @@ -2861,14 +2878,7 @@ void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruct Primitive::Type type = condition->InputAt(0)->GetType(); if (Primitive::IsFloatingPointType(type)) { - FPRegister lhs = InputFPRegisterAt(condition, 0); - if (condition->GetLocations()->InAt(1).IsConstant()) { - DCHECK(IsFloatingPointZeroConstant(condition->GetLocations()->InAt(1).GetConstant())); - // 0.0 is the only immediate that can be encoded directly in an FCMP instruction. - __ Fcmp(lhs, 0.0); - } else { - __ Fcmp(lhs, InputFPRegisterAt(condition, 1)); - } + GenerateFcmp(condition); if (true_target == nullptr) { IfCondition opposite_condition = condition->GetOppositeCondition(); __ B(ARM64FPCondition(opposite_condition, condition->IsGtBias()), false_target); @@ -3052,13 +3062,7 @@ void InstructionCodeGeneratorARM64::VisitSelect(HSelect* select) { csel_cond = HasSwappedInputs(variant) ? eq : ne; } } else if (IsConditionOnFloatingPointValues(cond)) { - Location rhs = cond->GetLocations()->InAt(1); - if (rhs.IsConstant()) { - DCHECK(IsFloatingPointZeroConstant(rhs.GetConstant())); - __ Fcmp(InputFPRegisterAt(cond, 0), 0.0); - } else { - __ Fcmp(InputFPRegisterAt(cond, 0), InputFPRegisterAt(cond, 1)); - } + GenerateFcmp(cond); csel_cond = GetConditionForSelect(cond->AsCondition(), variant); } else { __ Cmp(InputRegisterAt(cond, 0), InputOperandAt(cond, 1)); @@ -3665,23 +3669,21 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { // Add ADRP with its PC-relative DexCache access patch. - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetDexCacheArrayOffset()); - vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label; + const DexFile& dex_file = *invoke->GetTargetMethod().dex_file; + uint32_t element_offset = invoke->GetDexCacheArrayOffset(); + vixl::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); { vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); - __ Bind(pc_insn_label); - __ adrp(XRegisterFrom(temp), 0); + __ Bind(adrp_label); + __ adrp(XRegisterFrom(temp), /* offset placeholder */ 0); } - pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label; // Add LDR with its PC-relative DexCache access patch. - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetDexCacheArrayOffset()); + vixl::Label* ldr_label = + NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); { vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); - __ Bind(&pc_relative_dex_cache_patches_.back().label); - __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0)); - pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label; + __ Bind(ldr_label); + __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), /* offset placeholder */ 0)); } break; } @@ -3775,13 +3777,58 @@ void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location te __ Blr(lr); } +vixl::Label* CodeGeneratorARM64::NewPcRelativeStringPatch(const DexFile& dex_file, + uint32_t string_index, + vixl::Label* adrp_label) { + return NewPcRelativePatch(dex_file, string_index, adrp_label, &pc_relative_string_patches_); +} + +vixl::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset, + vixl::Label* adrp_label) { + return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_); +} + +vixl::Label* CodeGeneratorARM64::NewPcRelativePatch(const DexFile& dex_file, + uint32_t offset_or_index, + vixl::Label* adrp_label, + ArenaDeque<PcRelativePatchInfo>* patches) { + // Add a patch entry and return the label. + patches->emplace_back(dex_file, offset_or_index); + PcRelativePatchInfo* info = &patches->back(); + vixl::Label* label = &info->label; + // If adrp_label is null, this is the ADRP patch and needs to point to its own label. + info->pc_insn_label = (adrp_label != nullptr) ? adrp_label : label; + return label; +} + +vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageStringLiteral( + const DexFile& dex_file, uint32_t string_index) { + return boot_image_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); +} + +vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(uint64_t address) { + bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); + Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); +} + +vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral(uint64_t address) { + return DeduplicateUint64Literal(address); +} + void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + call_patches_.size() + relative_call_patches_.size() + - pc_relative_dex_cache_patches_.size(); + pc_relative_dex_cache_patches_.size() + + boot_image_string_patches_.size() + + pc_relative_string_patches_.size() + + boot_image_address_patches_.size(); linker_patches->reserve(size); for (const auto& entry : method_patches_) { const MethodReference& target_method = entry.first; @@ -3802,38 +3849,51 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patc info.target_method.dex_file, info.target_method.dex_method_index)); } - for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) { + for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(), &info.target_dex_file, info.pc_insn_label->location(), - info.element_offset)); + info.offset_or_index)); + } + for (const auto& entry : boot_image_string_patches_) { + const StringReference& target_string = entry.first; + vixl::Literal<uint32_t>* literal = entry.second; + linker_patches->push_back(LinkerPatch::StringPatch(literal->offset(), + target_string.dex_file, + target_string.string_index)); + } + for (const PcRelativePatchInfo& info : pc_relative_string_patches_) { + linker_patches->push_back(LinkerPatch::RelativeStringPatch(info.label.location(), + &info.target_dex_file, + info.pc_insn_label->location(), + info.offset_or_index)); } + for (const auto& entry : boot_image_address_patches_) { + DCHECK(GetCompilerOptions().GetIncludePatchInformation()); + vixl::Literal<uint32_t>* literal = entry.second; + linker_patches->push_back(LinkerPatch::RecordPosition(literal->offset())); + } +} + +vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value, + Uint32ToLiteralMap* map) { + return map->GetOrCreate( + value, + [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); }); } vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) { - // Look up the literal for value. - auto lb = uint64_literals_.lower_bound(value); - if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) { - return lb->second; - } - // We don't have a literal for this value, insert a new one. - vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value); - uint64_literals_.PutBefore(lb, value, literal); - return literal; + return uint64_literals_.GetOrCreate( + value, + [this, value]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(value); }); } vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral( MethodReference target_method, MethodToLiteralMap* map) { - // Look up the literal for target_method. - auto lb = map->lower_bound(target_method); - if (lb != map->end() && !map->key_comp()(target_method, lb->first)) { - return lb->second; - } - // We don't have a literal for this method yet, insert a new one. - vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u); - map->PutBefore(lb, target_method, literal); - return literal; + return map->GetOrCreate( + target_method, + [this]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(/* placeholder */ 0u); }); } vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral( @@ -3958,28 +4018,135 @@ void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UN // Nothing to do, this is driven by the code generator. } +HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) { Location out_loc = load->GetLocations()->Out(); Register out = OutputRegister(load); - Register current_method = InputRegisterAt(load, 0); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!kEmitCompilerReadBarrier); + __ Ldr(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + return; // No dex cache slow path. + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + // Add ADRP with its PC-relative String patch. + const DexFile& dex_file = load->GetDexFile(); + uint32_t string_index = load->GetStringIndex(); + vixl::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index); + { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(adrp_label); + __ adrp(out.X(), /* offset placeholder */ 0); + } + // Add ADD with its PC-relative String patch. + vixl::Label* add_label = + codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label); + { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(add_label); + __ add(out.X(), out.X(), Operand(/* offset placeholder */ 0)); + } + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK(load->GetAddress() != 0u && IsUint<32>(load->GetAddress())); + __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(load->GetAddress())); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads + // that gives a 16KiB range. To try and reduce the number of literals if we load + // multiple strings, simply split the dex cache address to a 16KiB aligned base + // loaded from a literal and the remaining offset embedded in the load. + static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes."); + DCHECK_ALIGNED(load->GetAddress(), 4u); + constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2; + uint64_t base_address = load->GetAddress() & ~MaxInt<uint64_t>(offset_bits); + uint32_t offset = load->GetAddress() & MaxInt<uint64_t>(offset_bits); + __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address)); + GenerateGcRootFieldLoad(load, out_loc, out.X(), offset); + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + // Add ADRP with its PC-relative DexCache access patch. + const DexFile& dex_file = load->GetDexFile(); + uint32_t element_offset = load->GetDexCacheElementOffset(); + vixl::Label* adrp_label = codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); + { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(adrp_label); + __ adrp(out.X(), /* offset placeholder */ 0); + } + // Add LDR with its PC-relative DexCache access patch. + vixl::Label* ldr_label = + codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); + GenerateGcRootFieldLoad(load, out_loc, out.X(), /* offset placeholder */ 0, ldr_label); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + Register current_method = InputRegisterAt(load, 0); + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load); @@ -4794,7 +4961,8 @@ void InstructionCodeGeneratorARM64::GenerateReferenceLoadTwoRegisters(HInstructi void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instruction, Location root, vixl::Register obj, - uint32_t offset) { + uint32_t offset, + vixl::Label* fixup_label) { Register root_reg = RegisterFrom(root, Primitive::kPrimNot); if (kEmitCompilerReadBarrier) { if (kUseBakerReadBarrier) { @@ -4807,7 +4975,13 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instru // } // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ Ldr(root_reg, MemOperand(obj, offset)); + if (fixup_label == nullptr) { + __ Ldr(root_reg, MemOperand(obj, offset)); + } else { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(fixup_label); + __ ldr(root_reg, MemOperand(obj, offset)); + } static_assert( sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " @@ -4832,14 +5006,26 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instru // GC root loaded through a slow path for read barriers other // than Baker's. // /* GcRoot<mirror::Object>* */ root = obj + offset - __ Add(root_reg.X(), obj.X(), offset); + if (fixup_label == nullptr) { + __ Add(root_reg.X(), obj.X(), offset); + } else { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(fixup_label); + __ add(root_reg.X(), obj.X(), offset); + } // /* mirror::Object* */ root = root->Read() codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ Ldr(root_reg, MemOperand(obj, offset)); + if (fixup_label == nullptr) { + __ Ldr(root_reg, MemOperand(obj, offset)); + } else { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(fixup_label); + __ ldr(root_reg, MemOperand(obj, offset)); + } // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index a1f686e39d..8ec753159a 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -17,6 +17,7 @@ #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ +#include "arch/arm64/quick_method_frame_info_arm64.h" #include "code_generator.h" #include "common_arm64.h" #include "dex/compiler_enums.h" @@ -24,9 +25,9 @@ #include "nodes.h" #include "parallel_move_resolver.h" #include "utils/arm64/assembler_arm64.h" +#include "utils/string_reference.h" #include "vixl/a64/disasm-a64.h" #include "vixl/a64/macro-assembler-a64.h" -#include "arch/arm64/quick_method_frame_info_arm64.h" namespace art { namespace arm64 { @@ -255,7 +256,11 @@ class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, vixl::Register obj, - uint32_t offset); + uint32_t offset, + vixl::Label* fixup_label = nullptr); + + // Generate a floating-point comparison. + void GenerateFcmp(HInstruction* instruction); void HandleShift(HBinaryOperation* instr); void GenerateTestAndBranch(HInstruction* instruction, @@ -450,6 +455,11 @@ class CodeGeneratorARM64 : public CodeGenerator { return false; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -464,6 +474,27 @@ class CodeGeneratorARM64 : public CodeGenerator { UNIMPLEMENTED(FATAL); } + // Add a new PC-relative string patch for an instruction and return the label + // to be bound before the instruction. The instruction will be either the + // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing + // to the associated ADRP patch label). + vixl::Label* NewPcRelativeStringPatch(const DexFile& dex_file, + uint32_t string_index, + vixl::Label* adrp_label = nullptr); + + // Add a new PC-relative dex cache array patch for an instruction and return + // the label to be bound before the instruction. The instruction will be + // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` + // pointing to the associated ADRP patch label). + vixl::Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset, + vixl::Label* adrp_label = nullptr); + + vixl::Literal<uint32_t>* DeduplicateBootImageStringLiteral(const DexFile& dex_file, + uint32_t string_index); + vixl::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address); + vixl::Literal<uint64_t>* DeduplicateDexCacheAddressLiteral(uint64_t address); + void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; // Fast path implementation of ReadBarrier::Barrier for a heap @@ -551,26 +582,39 @@ class CodeGeneratorARM64 : public CodeGenerator { bool use_load_acquire); using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::Literal<uint64_t>*>; + using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::Literal<uint32_t>*>; using MethodToLiteralMap = ArenaSafeMap<MethodReference, vixl::Literal<uint64_t>*, MethodReferenceComparator>; + using BootStringToLiteralMap = ArenaSafeMap<StringReference, + vixl::Literal<uint32_t>*, + StringReferenceValueComparator>; + vixl::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); vixl::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value); vixl::Literal<uint64_t>* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); vixl::Literal<uint64_t>* DeduplicateMethodAddressLiteral(MethodReference target_method); vixl::Literal<uint64_t>* DeduplicateMethodCodeLiteral(MethodReference target_method); - struct PcRelativeDexCacheAccessInfo { - PcRelativeDexCacheAccessInfo(const DexFile& dex_file, uint32_t element_off) - : target_dex_file(dex_file), element_offset(element_off), label(), pc_insn_label() { } + // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays + // and boot image strings. The only difference is the interpretation of the offset_or_index. + struct PcRelativePatchInfo { + PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) + : target_dex_file(dex_file), offset_or_index(off_or_idx), label(), pc_insn_label() { } const DexFile& target_dex_file; - uint32_t element_offset; + // Either the dex cache array element offset or the string index. + uint32_t offset_or_index; vixl::Label label; vixl::Label* pc_insn_label; }; + vixl::Label* NewPcRelativePatch(const DexFile& dex_file, + uint32_t offset_or_index, + vixl::Label* adrp_label, + ArenaDeque<PcRelativePatchInfo>* patches); + void EmitJumpTables(); // Labels for each block that will be compiled. @@ -584,7 +628,10 @@ class CodeGeneratorARM64 : public CodeGenerator { Arm64Assembler assembler_; const Arm64InstructionSetFeatures& isa_features_; - // Deduplication map for 64-bit literals, used for non-patchable method address and method code. + // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. + Uint32ToLiteralMap uint32_literals_; + // Deduplication map for 64-bit literals, used for non-patchable method address, method code + // or string dex cache address. Uint64ToLiteralMap uint64_literals_; // Method patch info, map MethodReference to a literal for method address and method code. MethodToLiteralMap method_patches_; @@ -593,7 +640,13 @@ class CodeGeneratorARM64 : public CodeGenerator { // Using ArenaDeque<> which retains element addresses on push/emplace_back(). ArenaDeque<MethodPatchInfo<vixl::Label>> relative_call_patches_; // PC-relative DexCache access info. - ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_; + ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; + // Deduplication map for boot string literals for kBootImageLinkTimeAddress. + BootStringToLiteralMap boot_image_string_patches_; + // PC-relative String patch info. + ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; + // Deduplication map for patchable boot image addresses. + Uint32ToLiteralMap boot_image_address_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); }; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 4c9206320f..8b19f84e1c 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -1455,9 +1455,8 @@ void InstructionCodeGeneratorMIPS::HandleShift(HBinaryOperation* instr) { bool use_imm = rhs_location.IsConstant(); Register rhs_reg = use_imm ? ZERO : rhs_location.AsRegister<Register>(); int64_t rhs_imm = use_imm ? CodeGenerator::GetInt64ValueOf(rhs_location.GetConstant()) : 0; - const uint32_t shift_mask = (type == Primitive::kPrimInt) - ? kMaxIntShiftValue - : kMaxLongShiftValue; + const uint32_t shift_mask = + (type == Primitive::kPrimInt) ? kMaxIntShiftDistance : kMaxLongShiftDistance; const uint32_t shift_value = rhs_imm & shift_mask; // Are the INS (Insert Bit Field) and ROTR instructions supported? bool has_ins_rotr = codegen_->GetInstructionSetFeatures().IsMipsIsaRevGreaterThanEqual2(); @@ -3272,13 +3271,13 @@ void InstructionCodeGeneratorMIPS::GenerateTestAndBranch(HInstruction* instructi // Nothing to do. The code always falls through. return; } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against 1. - if (cond->AsIntConstant()->IsOne()) { + // Constant condition, statically compared against "true" (integer value 1). + if (cond->AsIntConstant()->IsTrue()) { if (true_target != nullptr) { __ B(true_target); } } else { - DCHECK(cond->AsIntConstant()->IsZero()); + DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); if (false_target != nullptr) { __ B(false_target); } @@ -3817,6 +3816,12 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorMIPS* codegen return false; } +HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) { + // TODO: Implement other kinds. + return HLoadString::LoadKind::kDexCacheViaMethod; +} + HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, MethodReference target_method ATTRIBUTE_UNUSED) { @@ -4067,9 +4072,9 @@ void InstructionCodeGeneratorMIPS::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNU } void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->IsInDexCache() - ? LocationSummary::kNoCall - : LocationSummary::kCallOnSlowPath; + LocationSummary::CallKind call_kind = load->NeedsEnvironment() + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::RequiresRegister()); diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index b720573897..afe7917cc4 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -345,6 +345,11 @@ class CodeGeneratorMIPS : public CodeGenerator { return type == Primitive::kPrimLong; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 955c9a490e..2f9eca6ac3 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -1208,9 +1208,8 @@ void InstructionCodeGeneratorMIPS64::HandleShift(HBinaryOperation* instr) { } if (use_imm) { - uint32_t shift_value = (type == Primitive::kPrimInt) - ? static_cast<uint32_t>(rhs_imm & kMaxIntShiftValue) - : static_cast<uint32_t>(rhs_imm & kMaxLongShiftValue); + uint32_t shift_value = rhs_imm & + (type == Primitive::kPrimInt ? kMaxIntShiftDistance : kMaxLongShiftDistance); if (shift_value == 0) { if (dst != lhs) { @@ -2596,13 +2595,13 @@ void InstructionCodeGeneratorMIPS64::GenerateTestAndBranch(HInstruction* instruc // Nothing to do. The code always falls through. return; } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against 1. - if (cond->AsIntConstant()->IsOne()) { + // Constant condition, statically compared against "true" (integer value 1). + if (cond->AsIntConstant()->IsTrue()) { if (true_target != nullptr) { __ Bc(true_target); } } else { - DCHECK(cond->AsIntConstant()->IsZero()); + DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); if (false_target != nullptr) { __ Bc(false_target); } @@ -3031,6 +3030,12 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorMIPS64* codeg return false; } +HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) { + // TODO: Implement other kinds. + return HLoadString::LoadKind::kDexCacheViaMethod; +} + HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS64::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, MethodReference target_method ATTRIBUTE_UNUSED) { @@ -3285,9 +3290,9 @@ void InstructionCodeGeneratorMIPS64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_U } void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->IsInDexCache() - ? LocationSummary::kNoCall - : LocationSummary::kCallOnSlowPath; + LocationSummary::CallKind call_kind = load->NeedsEnvironment() + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::RequiresRegister()); diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 9464a140ad..94767cba4b 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -337,6 +337,11 @@ class CodeGeneratorMIPS64 : public CodeGenerator { bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const { return false; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index fb3216c895..715b5be2c8 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -799,6 +799,8 @@ CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Use a fake return address register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister)); @@ -1391,13 +1393,13 @@ void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instructio // Nothing to do. The code always falls through. return; } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against 1. - if (cond->AsIntConstant()->IsOne()) { + // Constant condition, statically compared against "true" (integer value 1). + if (cond->AsIntConstant()->IsTrue()) { if (true_target != nullptr) { __ jmp(true_target); } } else { - DCHECK(cond->AsIntConstant()->IsZero()); + DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); if (false_target != nullptr) { __ jmp(false_target); } @@ -3775,7 +3777,7 @@ void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) { __ shrl(first_reg, second_reg); } } else { - int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue; + int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftDistance; if (shift == 0) { return; } @@ -3803,7 +3805,7 @@ void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) { } } else { // Shift by a constant. - int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue; + int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftDistance; // Nothing to do if the shift is 0, as the input is already the output. if (shift != 0) { if (op->IsShl()) { @@ -3960,7 +3962,7 @@ void InstructionCodeGeneratorX86::VisitRor(HRor* ror) { Register second_reg = second.AsRegister<Register>(); __ rorl(first_reg, second_reg); } else { - Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue); + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftDistance); __ rorl(first_reg, imm); } return; @@ -3981,7 +3983,7 @@ void InstructionCodeGeneratorX86::VisitRor(HRor* ror) { __ cmovl(kNotEqual, first_reg_hi, first_reg_lo); __ cmovl(kNotEqual, first_reg_lo, temp_reg); } else { - int32_t shift_amt = CodeGenerator::GetInt64ValueOf(second.GetConstant()) & kMaxLongShiftValue; + int32_t shift_amt = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftDistance; if (shift_amt == 0) { // Already fine. return; @@ -4340,6 +4342,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOr // We disable pc-relative load when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many X86ComputeBaseMethodAddress instructions + // as needed for methods with irreducible loops. if (GetGraph()->HasIrreducibleLoops() && (dispatch_info.method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { @@ -4401,18 +4405,17 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress())); break; case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - __ movl(temp.AsRegister<Register>(), Immediate(0)); // Placeholder. + __ movl(temp.AsRegister<Register>(), Immediate(/* placeholder */ 0)); method_patches_.emplace_back(invoke->GetTargetMethod()); __ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn. break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, temp.AsRegister<Register>()); - uint32_t offset = invoke->GetDexCacheArrayOffset(); __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset)); - // Add the patch entry and bind its label at the end of the instruction. - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset); - __ Bind(&pc_relative_dex_cache_patches_.back().label); + // Bind a new fixup label at the end of the "movl" insn. + uint32_t offset = invoke->GetDexCacheArrayOffset(); + __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset)); break; } case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { @@ -4494,12 +4497,33 @@ void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); } +void CodeGeneratorX86::RecordSimplePatch() { + if (GetCompilerOptions().GetIncludePatchInformation()) { + simple_patches_.emplace_back(); + __ Bind(&simple_patches_.back()); + } +} + +void CodeGeneratorX86::RecordStringPatch(HLoadString* load_string) { + string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex()); + __ Bind(&string_patches_.back().label); +} + +Label* CodeGeneratorX86::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset) { + // Add the patch entry and bind its label at the end of the instruction. + pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset); + return &pc_relative_dex_cache_patches_.back().label; +} + void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + relative_call_patches_.size() + - pc_relative_dex_cache_patches_.size(); + pc_relative_dex_cache_patches_.size() + + simple_patches_.size() + + string_patches_.size(); linker_patches->reserve(size); // The label points to the end of the "movl" insn but the literal offset for method // patch needs to point to the embedded constant which occupies the last 4 bytes. @@ -4523,6 +4547,26 @@ void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche GetMethodAddressOffset(), info.element_offset)); } + for (const Label& label : simple_patches_) { + uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } + if (GetCompilerOptions().GetCompilePic()) { + for (const StringPatchInfo<Label>& info : string_patches_) { + uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset, + &info.dex_file, + GetMethodAddressOffset(), + info.string_index)); + } + } else { + for (const StringPatchInfo<Label>& info : string_patches_) { + uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, + &info.dex_file, + info.string_index)); + } + } } void CodeGeneratorX86::MarkGCCard(Register temp, @@ -5916,14 +5960,15 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) { DCHECK(!cls->MustGenerateClinitCheck()); // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ GenerateGcRootFieldLoad( - cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); } else { // /* GcRoot<mirror::Class>[] */ out = // current_method.ptr_sized_fields_->dex_cache_resolved_types_ __ movl(out, Address(current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value())); // /* GcRoot<mirror::Class> */ out = out[type_index] - GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); + GenerateGcRootFieldLoad( + cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()))); if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); @@ -5972,12 +6017,58 @@ void InstructionCodeGeneratorX86::GenerateClassInitializationCheck( // No need for memory fence, thanks to the X86 memory model. } +HLoadString::LoadKind CodeGeneratorX86::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + FALLTHROUGH_INTENDED; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); // Note: boot image is also non-JIT. + // We disable pc-relative load when there is an irreducible loop, as the optimization + // is incompatible with it. + // TODO: Create as many X86ComputeBaseMethodAddress instructions as needed for methods + // with irreducible loops. + if (GetGraph()->HasIrreducibleLoops()) { + return HLoadString::LoadKind::kDexCacheViaMethod; + } + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderX86::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + HLoadString::LoadKind load_kind = load->GetLoadKind(); + if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod || + load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative || + load_kind == HLoadString::LoadKind::kDexCachePcRelative) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } @@ -5985,16 +6076,61 @@ void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); - Register current_method = locations->InAt(0).AsRegister<Register>(); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value())); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: { + DCHECK(!kEmitCompilerReadBarrier); + __ movl(out, Immediate(/* placeholder */ 0)); + codegen_->RecordStringPatch(load); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + Register method_address = locations->InAt(0).AsRegister<Register>(); + __ leal(out, Address(method_address, CodeGeneratorX86::kDummy32BitOffset)); + codegen_->RecordStringPatch(load); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ movl(out, Immediate(address)); + codegen_->RecordSimplePatch(); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + GenerateGcRootFieldLoad(load, out_loc, Address::Absolute(address)); + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + Register base_reg = locations->InAt(0).AsRegister<Register>(); + uint32_t offset = load->GetDexCacheElementOffset(); + Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset); + GenerateGcRootFieldLoad( + load, out_loc, Address(base_reg, CodeGeneratorX86::kDummy32BitOffset), fixup_label); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + Register current_method = locations->InAt(0).AsRegister<Register>(); + + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); + + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value())); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex()))); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load); @@ -6692,21 +6828,24 @@ void InstructionCodeGeneratorX86::GenerateReferenceLoadTwoRegisters(HInstruction void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - Register obj, - uint32_t offset) { + const Address& address, + Label* fixup_label) { Register root_reg = root.AsRegister<Register>(); if (kEmitCompilerReadBarrier) { if (kUseBakerReadBarrier) { // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = obj.field; + // root = *address; // if (Thread::Current()->GetIsGcMarking()) { // root = ReadBarrier::Mark(root) // } - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } static_assert( sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " @@ -6727,15 +6866,21 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruct } else { // GC root loaded through a slow path for read barriers other // than Baker's. - // /* GcRoot<mirror::Object>* */ root = obj + offset - __ leal(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object>* */ root = address + __ leal(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // /* mirror::Object* */ root = root->Read() codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index c397899892..1fa22fcfbe 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -258,13 +258,13 @@ class InstructionCodeGeneratorX86 : public InstructionCodeGenerator { Location maybe_temp); // Generate a GC root reference load: // - // root <- *(obj + offset) + // root <- *address // // while honoring read barriers (if any). void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - Register obj, - uint32_t offset); + const Address& address, + Label* fixup_label = nullptr); // Push value to FPU stack. `is_fp` specifies whether the value is floating point or not. // `is_wide` specifies whether it is long/double or not. @@ -388,6 +388,11 @@ class CodeGeneratorX86 : public CodeGenerator { // Helper method to move a 64bits value between two locations. void Move64(Location destination, Location source); + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -399,6 +404,10 @@ class CodeGeneratorX86 : public CodeGenerator { // Generate a call to a virtual method. void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void RecordSimplePatch(); + void RecordStringPatch(HLoadString* load_string); + Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); + void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; // Emit linker patches. @@ -542,6 +551,10 @@ class CodeGeneratorX86 : public CodeGenerator { void GenerateImplicitNullCheck(HNullCheck* instruction); void GenerateExplicitNullCheck(HNullCheck* instruction); + // When we don't know the proper offset for the value, we use kDummy32BitOffset. + // The correct value will be inserted when processing Assembler fixups. + static constexpr int32_t kDummy32BitOffset = 256; + private: // Factored implementation of GenerateFieldLoadWithBakerReadBarrier // and GenerateArrayLoadWithBakerReadBarrier. @@ -578,6 +591,10 @@ class CodeGeneratorX86 : public CodeGenerator { ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_; // PC-relative DexCache access info. ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_; + // Patch locations for patchoat where the linker doesn't do any other work. + ArenaDeque<Label> simple_patches_; + // String patch locations. + ArenaDeque<StringPatchInfo<Label>> string_patches_; // Offset to the start of the constant area in the assembled code. // Used for fixups to the constant area. @@ -592,10 +609,6 @@ class CodeGeneratorX86 : public CodeGenerator { // instruction gives the address of the start of this method. int32_t method_address_offset_; - // When we don't know the proper offset for the value, we use kDummy32BitOffset. - // The correct value will be inserted when processing Assembler fixups. - static constexpr int32_t kDummy32BitOffset = 256; - DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86); }; diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 7648f61e75..cc46a07dcb 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -784,14 +784,14 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo method_patches_.emplace_back(invoke->GetTargetMethod()); __ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn. break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetDexCacheArrayOffset()); + case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { __ movq(temp.AsRegister<CpuRegister>(), Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); - // Bind the label at the end of the "movl" insn. - __ Bind(&pc_relative_dex_cache_patches_.back().label); + // Bind a new fixup label at the end of the "movl" insn. + uint32_t offset = invoke->GetDexCacheArrayOffset(); + __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset)); break; + } case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); Register method_reg; @@ -873,12 +873,33 @@ void CodeGeneratorX86_64::GenerateVirtualCall(HInvokeVirtual* invoke, Location t kX86_64WordSize).SizeValue())); } +void CodeGeneratorX86_64::RecordSimplePatch() { + if (GetCompilerOptions().GetIncludePatchInformation()) { + simple_patches_.emplace_back(); + __ Bind(&simple_patches_.back()); + } +} + +void CodeGeneratorX86_64::RecordStringPatch(HLoadString* load_string) { + string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex()); + __ Bind(&string_patches_.back().label); +} + +Label* CodeGeneratorX86_64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset) { + // Add a patch entry and return the label. + pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset); + return &pc_relative_dex_cache_patches_.back().label; +} + void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + relative_call_patches_.size() + - pc_relative_dex_cache_patches_.size(); + pc_relative_dex_cache_patches_.size() + + simple_patches_.size() + + string_patches_.size(); linker_patches->reserve(size); // The label points to the end of the "movl" insn but the literal offset for method // patch needs to point to the embedded constant which occupies the last 4 bytes. @@ -902,6 +923,18 @@ void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pat info.label.Position(), info.element_offset)); } + for (const Label& label : simple_patches_) { + uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } + for (const StringPatchInfo<Label>& info : string_patches_) { + // These are always PC-relative, see GetSupportedLoadStringKind(). + uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset, + &info.dex_file, + info.label.Position(), + info.string_index)); + } } void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const { @@ -978,6 +1011,8 @@ CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph, method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister)); } @@ -1402,13 +1437,13 @@ void InstructionCodeGeneratorX86_64::GenerateTestAndBranch(HInstruction* instruc // Nothing to do. The code always falls through. return; } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against 1. - if (cond->AsIntConstant()->IsOne()) { + // Constant condition, statically compared against "true" (integer value 1). + if (cond->AsIntConstant()->IsTrue()) { if (true_target != nullptr) { __ jmp(true_target); } } else { - DCHECK(cond->AsIntConstant()->IsZero()); + DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); if (false_target != nullptr) { __ jmp(false_target); } @@ -2799,7 +2834,7 @@ void InstructionCodeGeneratorX86_64::VisitTypeConversion(HTypeConversion* conver } else if (in.IsConstant()) { int64_t v = in.GetConstant()->AsLongConstant()->GetValue(); XmmRegister dest = out.AsFpuRegister<XmmRegister>(); - codegen_->Load64BitValue(dest, static_cast<double>(v)); + codegen_->Load32BitValue(dest, static_cast<float>(v)); } else { __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), in.GetStackIndex()), true); @@ -3799,7 +3834,7 @@ void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) { __ shrl(first_reg, second_reg); } } else { - Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue); + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftDistance); if (op->IsShl()) { __ shll(first_reg, imm); } else if (op->IsShr()) { @@ -3821,7 +3856,7 @@ void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) { __ shrq(first_reg, second_reg); } } else { - Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue); + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftDistance); if (op->IsShl()) { __ shlq(first_reg, imm); } else if (op->IsShr()) { @@ -3868,7 +3903,7 @@ void InstructionCodeGeneratorX86_64::VisitRor(HRor* ror) { CpuRegister second_reg = second.AsRegister<CpuRegister>(); __ rorl(first_reg, second_reg); } else { - Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue); + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftDistance); __ rorl(first_reg, imm); } break; @@ -3877,7 +3912,7 @@ void InstructionCodeGeneratorX86_64::VisitRor(HRor* ror) { CpuRegister second_reg = second.AsRegister<CpuRegister>(); __ rorq(first_reg, second_reg); } else { - Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue); + Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftDistance); __ rorq(first_reg, imm); } break; @@ -5249,6 +5284,12 @@ void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) { CpuRegister(ensure_scratch.GetRegister())); } +void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg1, CpuRegister reg2) { + __ movq(CpuRegister(TMP), reg1); + __ movq(reg1, reg2); + __ movq(reg2, CpuRegister(TMP)); +} + void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) { __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem)); __ movq(Address(CpuRegister(RSP), mem), reg); @@ -5286,7 +5327,7 @@ void ParallelMoveResolverX86_64::EmitSwap(size_t index) { Location destination = move->GetDestination(); if (source.IsRegister() && destination.IsRegister()) { - __ xchgq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>()); + Exchange64(source.AsRegister<CpuRegister>(), destination.AsRegister<CpuRegister>()); } else if (source.IsRegister() && destination.IsStackSlot()) { Exchange32(source.AsRegister<CpuRegister>(), destination.GetStackIndex()); } else if (source.IsStackSlot() && destination.IsRegister()) { @@ -5365,14 +5406,15 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) { DCHECK(!cls->MustGenerateClinitCheck()); // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ GenerateGcRootFieldLoad( - cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); } else { // /* GcRoot<mirror::Class>[] */ out = // current_method.ptr_sized_fields_->dex_cache_resolved_types_ __ movq(out, Address(current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86_64PointerSize).Int32Value())); // /* GcRoot<mirror::Class> */ out = out[type_index] - GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); + GenerateGcRootFieldLoad( + cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()))); if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); @@ -5410,12 +5452,49 @@ void InstructionCodeGeneratorX86_64::VisitClinitCheck(HClinitCheck* check) { check->GetLocations()->InAt(0).AsRegister<CpuRegister>()); } +HLoadString::LoadKind CodeGeneratorX86_64::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + // We prefer the always-available RIP-relative address for the x86-64 boot image. + return HLoadString::LoadKind::kBootImageLinkTimePcRelative; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } @@ -5423,16 +5502,59 @@ void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); CpuRegister out = out_loc.AsRegister<CpuRegister>(); - CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>(); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + __ leal(out, Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, /* no_rip */ false)); + codegen_->RecordStringPatch(load); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ movl(out, Immediate(address)); // Zero-extended. + codegen_->RecordSimplePatch(); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + if (IsUint<32>(load->GetAddress())) { + Address address = Address::Absolute(load->GetAddress(), /* no_rip */ true); + GenerateGcRootFieldLoad(load, out_loc, address); + } else { + // TODO: Consider using opcode A1, i.e. movl eax, moff32 (with 64-bit address). + __ movq(out, Immediate(load->GetAddress())); + GenerateGcRootFieldLoad(load, out_loc, Address(out, 0)); + } + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + uint32_t offset = load->GetDexCacheElementOffset(); + Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset); + Address address = Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, + /* no_rip */ false); + GenerateGcRootFieldLoad(load, out_loc, address, fixup_label); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>(); + + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex()))); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load); @@ -6171,21 +6293,24 @@ void InstructionCodeGeneratorX86_64::GenerateReferenceLoadTwoRegisters(HInstruct void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - CpuRegister obj, - uint32_t offset) { + const Address& address, + Label* fixup_label) { CpuRegister root_reg = root.AsRegister<CpuRegister>(); if (kEmitCompilerReadBarrier) { if (kUseBakerReadBarrier) { // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = obj.field; + // root = *address; // if (Thread::Current()->GetIsGcMarking()) { // root = ReadBarrier::Mark(root) // } - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } static_assert( sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " @@ -6207,15 +6332,21 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instr } else { // GC root loaded through a slow path for read barriers other // than Baker's. - // /* GcRoot<mirror::Object>* */ root = obj + offset - __ leaq(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object>* */ root = address + __ leaq(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // /* mirror::Object* */ root = root->Read() codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index c3fce6e824..7ebce58f6c 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -142,6 +142,7 @@ class ParallelMoveResolverX86_64 : public ParallelMoveResolverWithSwap { void Exchange32(CpuRegister reg, int mem); void Exchange32(XmmRegister reg, int mem); void Exchange32(int mem1, int mem2); + void Exchange64(CpuRegister reg1, CpuRegister reg2); void Exchange64(CpuRegister reg, int mem); void Exchange64(XmmRegister reg, int mem); void Exchange64(int mem1, int mem2); @@ -252,13 +253,13 @@ class InstructionCodeGeneratorX86_64 : public InstructionCodeGenerator { Location maybe_temp); // Generate a GC root reference load: // - // root <- *(obj + offset) + // root <- *address // // while honoring read barriers (if any). void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - CpuRegister obj, - uint32_t offset); + const Address& address, + Label* fixup_label = nullptr); void PushOntoFPStack(Location source, uint32_t temp_offset, uint32_t stack_adjustment, bool is_float); @@ -384,6 +385,11 @@ class CodeGeneratorX86_64 : public CodeGenerator { return false; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -393,6 +399,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void RecordSimplePatch(); + void RecordStringPatch(HLoadString* load_string); + Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); + void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; @@ -515,6 +525,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { void GenerateImplicitNullCheck(HNullCheck* instruction); void GenerateExplicitNullCheck(HNullCheck* instruction); + // When we don't know the proper offset for the value, we use kDummy32BitOffset. + // We will fix this up in the linker later to have the right value. + static constexpr int32_t kDummy32BitOffset = 256; + private: // Factored implementation of GenerateFieldLoadWithBakerReadBarrier // and GenerateArrayLoadWithBakerReadBarrier. @@ -552,10 +566,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_; // PC-relative DexCache access info. ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_; - - // When we don't know the proper offset for the value, we use kDummy32BitOffset. - // We will fix this up in the linker later to have the right value. - static constexpr int32_t kDummy32BitOffset = 256; + // Patch locations for patchoat where the linker doesn't do any other work. + ArenaDeque<Label> simple_patches_; + // String patch locations. + ArenaDeque<StringPatchInfo<Label>> string_patches_; // Fixups for jump tables need to be handled specially. ArenaVector<JumpTableRIPFixup*> fixups_to_jump_tables_; diff --git a/compiler/optimizing/constant_folding.cc b/compiler/optimizing/constant_folding.cc index 7ddabdee78..0614945ddc 100644 --- a/compiler/optimizing/constant_folding.cc +++ b/compiler/optimizing/constant_folding.cc @@ -120,7 +120,7 @@ void HConstantFoldingVisitor::VisitTypeConversion(HTypeConversion* inst) { void HConstantFoldingVisitor::VisitDivZeroCheck(HDivZeroCheck* inst) { // We can safely remove the check if the input is a non-null constant. HInstruction* check_input = inst->InputAt(0); - if (check_input->IsConstant() && !check_input->AsConstant()->IsZero()) { + if (check_input->IsConstant() && !check_input->AsConstant()->IsArithmeticZero()) { inst->ReplaceWith(check_input); inst->GetBlock()->RemoveInstruction(inst); } @@ -130,7 +130,7 @@ void HConstantFoldingVisitor::VisitDivZeroCheck(HDivZeroCheck* inst) { void InstructionWithAbsorbingInputSimplifier::VisitShift(HBinaryOperation* instruction) { DCHECK(instruction->IsShl() || instruction->IsShr() || instruction->IsUShr()); HInstruction* left = instruction->GetLeft(); - if (left->IsConstant() && left->AsConstant()->IsZero()) { + if (left->IsConstant() && left->AsConstant()->IsArithmeticZero()) { // Replace code looking like // SHL dst, 0, shift_amount // with @@ -142,7 +142,7 @@ void InstructionWithAbsorbingInputSimplifier::VisitShift(HBinaryOperation* instr void InstructionWithAbsorbingInputSimplifier::VisitAbove(HAbove* instruction) { if (instruction->GetLeft()->IsConstant() && - instruction->GetLeft()->AsConstant()->IsZero()) { + instruction->GetLeft()->AsConstant()->IsArithmeticZero()) { // Replace code looking like // ABOVE dst, 0, src // unsigned 0 > src is always false // with @@ -154,7 +154,7 @@ void InstructionWithAbsorbingInputSimplifier::VisitAbove(HAbove* instruction) { void InstructionWithAbsorbingInputSimplifier::VisitAboveOrEqual(HAboveOrEqual* instruction) { if (instruction->GetRight()->IsConstant() && - instruction->GetRight()->AsConstant()->IsZero()) { + instruction->GetRight()->AsConstant()->IsArithmeticZero()) { // Replace code looking like // ABOVE_OR_EQUAL dst, src, 0 // unsigned src >= 0 is always true // with @@ -166,7 +166,7 @@ void InstructionWithAbsorbingInputSimplifier::VisitAboveOrEqual(HAboveOrEqual* i void InstructionWithAbsorbingInputSimplifier::VisitBelow(HBelow* instruction) { if (instruction->GetRight()->IsConstant() && - instruction->GetRight()->AsConstant()->IsZero()) { + instruction->GetRight()->AsConstant()->IsArithmeticZero()) { // Replace code looking like // BELOW dst, src, 0 // unsigned src < 0 is always false // with @@ -178,7 +178,7 @@ void InstructionWithAbsorbingInputSimplifier::VisitBelow(HBelow* instruction) { void InstructionWithAbsorbingInputSimplifier::VisitBelowOrEqual(HBelowOrEqual* instruction) { if (instruction->GetLeft()->IsConstant() && - instruction->GetLeft()->AsConstant()->IsZero()) { + instruction->GetLeft()->AsConstant()->IsArithmeticZero()) { // Replace code looking like // BELOW_OR_EQUAL dst, 0, src // unsigned 0 <= src is always true // with @@ -190,7 +190,7 @@ void InstructionWithAbsorbingInputSimplifier::VisitBelowOrEqual(HBelowOrEqual* i void InstructionWithAbsorbingInputSimplifier::VisitAnd(HAnd* instruction) { HConstant* input_cst = instruction->GetConstantRight(); - if ((input_cst != nullptr) && input_cst->IsZero()) { + if ((input_cst != nullptr) && input_cst->IsZeroBitPattern()) { // Replace code looking like // AND dst, src, 0 // with @@ -224,7 +224,7 @@ void InstructionWithAbsorbingInputSimplifier::VisitMul(HMul* instruction) { HConstant* input_cst = instruction->GetConstantRight(); Primitive::Type type = instruction->GetType(); if (Primitive::IsIntOrLongType(type) && - (input_cst != nullptr) && input_cst->IsZero()) { + (input_cst != nullptr) && input_cst->IsArithmeticZero()) { // Replace code looking like // MUL dst, src, 0 // with @@ -264,7 +264,7 @@ void InstructionWithAbsorbingInputSimplifier::VisitRem(HRem* instruction) { HBasicBlock* block = instruction->GetBlock(); if (instruction->GetLeft()->IsConstant() && - instruction->GetLeft()->AsConstant()->IsZero()) { + instruction->GetLeft()->AsConstant()->IsArithmeticZero()) { // Replace code looking like // REM dst, 0, src // with diff --git a/compiler/optimizing/constant_folding_test.cc b/compiler/optimizing/constant_folding_test.cc index 9c69f8c75b..1e54a0adfa 100644 --- a/compiler/optimizing/constant_folding_test.cc +++ b/compiler/optimizing/constant_folding_test.cc @@ -598,42 +598,41 @@ TEST_F(ConstantFoldingTest, IntConstantFoldingAndJumps) { " 5: IntConstant [9]\n" // v1 <- 2 " 13: IntConstant [14]\n" // const 5 " 18: IntConstant [19]\n" // const 4 - " 24: IntConstant [25]\n" // const 8 - " 30: SuspendCheck\n" - " 31: Goto 1\n" + " 23: IntConstant [24]\n" // const 8 + " 29: SuspendCheck\n" + " 30: Goto 1\n" "BasicBlock 1, pred: 0, succ: 3\n" " 9: Add(3, 5) [19]\n" // v2 <- v0 + v1 = 1 + 2 = 3 " 11: Goto 3\n" // goto L2 "BasicBlock 2, pred: 3, succ: 4\n" // L1: - " 14: Add(19, 13) [25]\n" // v1 <- v0 + 3 = 7 + 5 = 12 + " 14: Add(19, 13) [24]\n" // v1 <- v0 + 3 = 7 + 5 = 12 " 16: Goto 4\n" // goto L3 "BasicBlock 3, pred: 1, succ: 2\n" // L2: " 19: Add(9, 18) [14]\n" // v0 <- v2 + 2 = 3 + 4 = 7 - " 21: SuspendCheck\n" - " 22: Goto 2\n" // goto L1 + " 21: Goto 2\n" // goto L1 "BasicBlock 4, pred: 2, succ: 5\n" // L3: - " 25: Add(14, 24) [28]\n" // v2 <- v1 + 4 = 12 + 8 = 20 - " 28: Return(25)\n" // return v2 + " 24: Add(14, 23) [27]\n" // v2 <- v1 + 4 = 12 + 8 = 20 + " 27: Return(24)\n" // return v2 "BasicBlock 5, pred: 4\n" - " 29: Exit\n"; + " 28: Exit\n"; // Expected difference after constant folding. diff_t expected_cf_diff = { { " 3: IntConstant [9]\n", " 3: IntConstant\n" }, - { " 5: IntConstant [9]\n", " 5: IntConstant []\n" }, + { " 5: IntConstant [9]\n", " 5: IntConstant\n" }, { " 13: IntConstant [14]\n", " 13: IntConstant\n" }, { " 18: IntConstant [19]\n", " 18: IntConstant\n" }, - { " 24: IntConstant [25]\n", " 24: IntConstant\n" }, - { " 30: SuspendCheck\n", " 30: SuspendCheck\n" - " 32: IntConstant []\n" - " 33: IntConstant []\n" - " 34: IntConstant\n" - " 35: IntConstant [28]\n" }, + { " 23: IntConstant [24]\n", " 23: IntConstant\n" }, + { " 29: SuspendCheck\n", " 29: SuspendCheck\n" + " 31: IntConstant\n" + " 32: IntConstant\n" + " 33: IntConstant\n" + " 34: IntConstant [27]\n" }, { " 9: Add(3, 5) [19]\n", removed }, - { " 14: Add(19, 13) [25]\n", removed }, + { " 14: Add(19, 13) [24]\n", removed }, { " 19: Add(9, 18) [14]\n", removed }, - { " 25: Add(14, 24) [28]\n", removed }, - { " 28: Return(25)\n", " 28: Return(35)\n"} + { " 24: Add(14, 23) [27]\n", removed }, + { " 27: Return(24)\n", " 27: Return(34)\n"} }; std::string expected_after_cf = Patch(expected_before, expected_cf_diff); @@ -656,17 +655,13 @@ TEST_F(ConstantFoldingTest, IntConstantFoldingAndJumps) { // Expected difference after dead code elimination. std::string expected_after_dce = "BasicBlock 0, succ: 1\n" - " 5: IntConstant []\n" - " 30: SuspendCheck\n" - " 32: IntConstant []\n" - " 33: IntConstant []\n" - " 35: IntConstant [28]\n" - " 31: Goto 1\n" + " 29: SuspendCheck\n" + " 34: IntConstant [27]\n" + " 30: Goto 1\n" "BasicBlock 1, pred: 0, succ: 5\n" - " 21: SuspendCheck\n" - " 28: Return(35)\n" + " 27: Return(34)\n" "BasicBlock 5, pred: 1\n" - " 29: Exit\n"; + " 28: Exit\n"; TestCode(data, expected_before, diff --git a/compiler/optimizing/dead_code_elimination.cc b/compiler/optimizing/dead_code_elimination.cc index d7bf16e0cc..5f11024996 100644 --- a/compiler/optimizing/dead_code_elimination.cc +++ b/compiler/optimizing/dead_code_elimination.cc @@ -41,11 +41,11 @@ static void MarkReachableBlocks(HGraph* graph, ArenaBitVector* visited) { HIf* if_instruction = last_instruction->AsIf(); HInstruction* condition = if_instruction->InputAt(0); if (condition->IsIntConstant()) { - if (condition->AsIntConstant()->IsOne()) { + if (condition->AsIntConstant()->IsTrue()) { live_successors = live_successors.SubArray(0u, 1u); DCHECK_EQ(live_successors[0], if_instruction->IfTrueSuccessor()); } else { - DCHECK(condition->AsIntConstant()->IsZero()); + DCHECK(condition->AsIntConstant()->IsFalse()) << condition->AsIntConstant()->GetValue(); live_successors = live_successors.SubArray(1u, 1u); DCHECK_EQ(live_successors[0], if_instruction->IfFalseSuccessor()); } diff --git a/compiler/optimizing/dead_code_elimination_test.cc b/compiler/optimizing/dead_code_elimination_test.cc index 930795b4f6..83e724ba29 100644 --- a/compiler/optimizing/dead_code_elimination_test.cc +++ b/compiler/optimizing/dead_code_elimination_test.cc @@ -149,44 +149,32 @@ TEST_F(DeadCodeEliminationTest, AdditionsAndInconditionalJumps) { " 5: IntConstant [9]\n" " 13: IntConstant [14]\n" " 18: IntConstant [19]\n" - " 24: IntConstant [25]\n" - " 29: SuspendCheck\n" - " 30: Goto 1\n" + " 23: IntConstant [24]\n" + " 28: SuspendCheck\n" + " 29: Goto 1\n" "BasicBlock 1, pred: 0, succ: 3\n" " 9: Add(3, 5) [19]\n" " 11: Goto 3\n" "BasicBlock 2, pred: 3, succ: 4\n" - " 14: Add(19, 13) [25]\n" + " 14: Add(19, 13) [24]\n" " 16: Goto 4\n" "BasicBlock 3, pred: 1, succ: 2\n" " 19: Add(9, 18) [14]\n" - " 21: SuspendCheck\n" - " 22: Goto 2\n" + " 21: Goto 2\n" "BasicBlock 4, pred: 2, succ: 5\n" - " 25: Add(14, 24)\n" - " 27: ReturnVoid\n" + " 24: Add(14, 23)\n" + " 26: ReturnVoid\n" "BasicBlock 5, pred: 4\n" - " 28: Exit\n"; + " 27: Exit\n"; - // The SuspendCheck instruction following this Add instruction - // inserts the latter in an environment, thus making it "used" and - // therefore non removable. It ensures that some other Add and - // IntConstant instructions cannot be removed, as they are direct - // or indirect inputs of the initial Add instruction. std::string expected_after = "BasicBlock 0, succ: 1\n" - " 3: IntConstant [9]\n" - " 5: IntConstant [9]\n" - " 18: IntConstant [19]\n" - " 29: SuspendCheck\n" - " 30: Goto 1\n" + " 28: SuspendCheck\n" + " 29: Goto 1\n" "BasicBlock 1, pred: 0, succ: 5\n" - " 9: Add(3, 5) [19]\n" - " 19: Add(9, 18) []\n" - " 21: SuspendCheck\n" - " 27: ReturnVoid\n" + " 26: ReturnVoid\n" "BasicBlock 5, pred: 1\n" - " 28: Exit\n"; + " 27: Exit\n"; TestCode(data, expected_before, expected_after); } diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.cc b/compiler/optimizing/dex_cache_array_fixups_arm.cc index 3db254a004..e9072b9c77 100644 --- a/compiler/optimizing/dex_cache_array_fixups_arm.cc +++ b/compiler/optimizing/dex_cache_array_fixups_arm.cc @@ -44,6 +44,21 @@ class DexCacheArrayFixupsVisitor : public HGraphVisitor { } private: + void VisitLoadString(HLoadString* load_string) OVERRIDE { + // If this is a load with PC-relative access to the dex cache methods array, + // we need to add the dex cache arrays base as the special input. + if (load_string->GetLoadKind() == HLoadString::LoadKind::kDexCachePcRelative) { + // Initialize base for target dex file if needed. + const DexFile& dex_file = load_string->GetDexFile(); + HArmDexCacheArraysBase* base = GetOrCreateDexCacheArrayBase(dex_file); + // Update the element offset in base. + DexCacheArraysLayout layout(kArmPointerSize, &dex_file); + base->UpdateElementOffset(layout.StringOffset(load_string->GetStringIndex())); + // Add the special argument base to the load. + load_string->AddSpecialInput(base); + } + } + void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { // If this is an invoke with PC-relative access to the dex cache methods array, // we need to add the dex cache arrays base as the special input. diff --git a/compiler/optimizing/graph_checker.cc b/compiler/optimizing/graph_checker.cc index 0c22903602..f2e77e28a6 100644 --- a/compiler/optimizing/graph_checker.cc +++ b/compiler/optimizing/graph_checker.cc @@ -32,11 +32,9 @@ void GraphChecker::VisitBasicBlock(HBasicBlock* block) { // Check consistency with respect to predecessors of `block`. // Note: Counting duplicates with a sorted vector uses up to 6x less memory - // than ArenaSafeMap<HBasicBlock*, size_t>. - ArenaVector<HBasicBlock*> sorted_predecessors( - block->GetPredecessors().begin(), - block->GetPredecessors().end(), - GetGraph()->GetArena()->Adapter(kArenaAllocGraphChecker)); + // than ArenaSafeMap<HBasicBlock*, size_t> and also allows storage reuse. + ArenaVector<HBasicBlock*>& sorted_predecessors = blocks_storage_; + sorted_predecessors.assign(block->GetPredecessors().begin(), block->GetPredecessors().end()); std::sort(sorted_predecessors.begin(), sorted_predecessors.end()); for (auto it = sorted_predecessors.begin(), end = sorted_predecessors.end(); it != end; ) { HBasicBlock* p = *it++; @@ -57,11 +55,9 @@ void GraphChecker::VisitBasicBlock(HBasicBlock* block) { // Check consistency with respect to successors of `block`. // Note: Counting duplicates with a sorted vector uses up to 6x less memory - // than ArenaSafeMap<HBasicBlock*, size_t>. - ArenaVector<HBasicBlock*> sorted_successors( - block->GetSuccessors().begin(), - block->GetSuccessors().end(), - GetGraph()->GetArena()->Adapter(kArenaAllocGraphChecker)); + // than ArenaSafeMap<HBasicBlock*, size_t> and also allows storage reuse. + ArenaVector<HBasicBlock*>& sorted_successors = blocks_storage_; + sorted_successors.assign(block->GetSuccessors().begin(), block->GetSuccessors().end()); std::sort(sorted_successors.begin(), sorted_successors.end()); for (auto it = sorted_successors.begin(), end = sorted_successors.end(); it != end; ) { HBasicBlock* s = *it++; @@ -811,10 +807,11 @@ void GraphChecker::VisitPhi(HPhi* phi) { phi->GetRegNumber(), type_str.str().c_str())); } else { - ArenaBitVector visited(GetGraph()->GetArena(), - 0, - /* expandable */ true, - kArenaAllocGraphChecker); + // If we get here, make sure we allocate all the necessary storage at once + // because the BitVector reallocation strategy has very bad worst-case behavior. + ArenaBitVector& visited = visited_storage_; + visited.SetBit(GetGraph()->GetCurrentInstructionId()); + visited.ClearAllBits(); if (!IsConstantEquivalent(phi, other_phi, &visited)) { AddError(StringPrintf("Two phis (%d and %d) found for VReg %d but they " "are not equivalents of constants.", @@ -840,17 +837,11 @@ void GraphChecker::HandleBooleanInput(HInstruction* instruction, size_t input_in static_cast<int>(input_index), value)); } - } else if (input->GetType() == Primitive::kPrimInt - && (input->IsPhi() || - input->IsAnd() || - input->IsOr() || - input->IsXor() || - input->IsSelect())) { - // TODO: We need a data-flow analysis to determine if the Phi or Select or - // binary operation is actually Boolean. Allow for now. - } else if (input->GetType() != Primitive::kPrimBoolean) { + } else if (Primitive::PrimitiveKind(input->GetType()) != Primitive::kPrimInt) { + // TODO: We need a data-flow analysis to determine if an input like Phi, + // Select or a binary operation is actually Boolean. Allow for now. AddError(StringPrintf( - "%s instruction %d has a non-Boolean input %d whose type is: %s.", + "%s instruction %d has a non-integer input %d whose type is: %s.", instruction->DebugName(), instruction->GetId(), static_cast<int>(input_index), @@ -937,9 +928,12 @@ void GraphChecker::VisitBinaryOperation(HBinaryOperation* op) { Primitive::Type lhs_type = op->InputAt(0)->GetType(); Primitive::Type rhs_type = op->InputAt(1)->GetType(); Primitive::Type result_type = op->GetType(); + + // Type consistency between inputs. if (op->IsUShr() || op->IsShr() || op->IsShl() || op->IsRor()) { if (Primitive::PrimitiveKind(rhs_type) != Primitive::kPrimInt) { - AddError(StringPrintf("Shift operation %s %d has a non-int kind second input: %s of type %s.", + AddError(StringPrintf("Shift/rotate operation %s %d has a non-int kind second input: " + "%s of type %s.", op->DebugName(), op->GetId(), op->InputAt(1)->DebugName(), Primitive::PrettyDescriptor(rhs_type))); @@ -953,21 +947,38 @@ void GraphChecker::VisitBinaryOperation(HBinaryOperation* op) { } } + // Type consistency between result and input(s). if (op->IsCompare()) { if (result_type != Primitive::kPrimInt) { AddError(StringPrintf("Compare operation %d has a non-int result type: %s.", op->GetId(), Primitive::PrettyDescriptor(result_type))); } + } else if (op->IsUShr() || op->IsShr() || op->IsShl() || op->IsRor()) { + // Only check the first input (value), as the second one (distance) + // must invariably be of kind `int`. + if (result_type != Primitive::PrimitiveKind(lhs_type)) { + AddError(StringPrintf("Shift/rotate operation %s %d has a result type different " + "from its left-hand side (value) input kind: %s vs %s.", + op->DebugName(), op->GetId(), + Primitive::PrettyDescriptor(result_type), + Primitive::PrettyDescriptor(lhs_type))); + } } else { - // Use the first input, so that we can also make this check for shift and rotate operations. if (Primitive::PrimitiveKind(result_type) != Primitive::PrimitiveKind(lhs_type)) { AddError(StringPrintf("Binary operation %s %d has a result kind different " - "from its input kind: %s vs %s.", + "from its left-hand side input kind: %s vs %s.", op->DebugName(), op->GetId(), Primitive::PrettyDescriptor(result_type), Primitive::PrettyDescriptor(lhs_type))); } + if (Primitive::PrimitiveKind(result_type) != Primitive::PrimitiveKind(rhs_type)) { + AddError(StringPrintf("Binary operation %s %d has a result kind different " + "from its right-hand side input kind: %s vs %s.", + op->DebugName(), op->GetId(), + Primitive::PrettyDescriptor(result_type), + Primitive::PrettyDescriptor(rhs_type))); + } } } diff --git a/compiler/optimizing/graph_checker.h b/compiler/optimizing/graph_checker.h index 27d5621887..a536b30fcd 100644 --- a/compiler/optimizing/graph_checker.h +++ b/compiler/optimizing/graph_checker.h @@ -33,7 +33,9 @@ class GraphChecker : public HGraphDelegateVisitor { seen_ids_(graph->GetArena(), graph->GetCurrentInstructionId(), false, - kArenaAllocGraphChecker) {} + kArenaAllocGraphChecker), + blocks_storage_(graph->GetArena()->Adapter(kArenaAllocGraphChecker)), + visited_storage_(graph->GetArena(), 0u, true, kArenaAllocGraphChecker) {} // Check the whole graph (in reverse post-order). void Run() { @@ -102,6 +104,10 @@ class GraphChecker : public HGraphDelegateVisitor { const char* const dump_prefix_; ArenaBitVector seen_ids_; + // To reduce the total arena memory allocation, we reuse the same storage. + ArenaVector<HBasicBlock*> blocks_storage_; + ArenaBitVector visited_storage_; + DISALLOW_COPY_AND_ASSIGN(GraphChecker); }; diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc index 3a9d242df2..fe47f7db7d 100644 --- a/compiler/optimizing/graph_visualizer.cc +++ b/compiler/optimizing/graph_visualizer.cc @@ -178,41 +178,47 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { : nullptr), indent_(0) {} + void Flush() { + // We use "\n" instead of std::endl to avoid implicit flushing which + // generates too many syscalls during debug-GC tests (b/27826765). + output_ << std::flush; + } + void StartTag(const char* name) { AddIndent(); - output_ << "begin_" << name << std::endl; + output_ << "begin_" << name << "\n"; indent_++; } void EndTag(const char* name) { indent_--; AddIndent(); - output_ << "end_" << name << std::endl; + output_ << "end_" << name << "\n"; } void PrintProperty(const char* name, const char* property) { AddIndent(); - output_ << name << " \"" << property << "\"" << std::endl; + output_ << name << " \"" << property << "\"\n"; } void PrintProperty(const char* name, const char* property, int id) { AddIndent(); - output_ << name << " \"" << property << id << "\"" << std::endl; + output_ << name << " \"" << property << id << "\"\n"; } void PrintEmptyProperty(const char* name) { AddIndent(); - output_ << name << std::endl; + output_ << name << "\n"; } void PrintTime(const char* name) { AddIndent(); - output_ << name << " " << time(nullptr) << std::endl; + output_ << name << " " << time(nullptr) << "\n"; } void PrintInt(const char* name, int value) { AddIndent(); - output_ << name << " " << value << std::endl; + output_ << name << " " << value << "\n"; } void AddIndent() { @@ -249,7 +255,7 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { if (block->IsEntryBlock() && (disasm_info_ != nullptr)) { output_ << " \"" << kDisassemblyBlockFrameEntry << "\" "; } - output_<< std::endl; + output_<< "\n"; } void PrintSuccessors(HBasicBlock* block) { @@ -258,7 +264,7 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { for (HBasicBlock* successor : block->GetNormalSuccessors()) { output_ << " \"B" << successor->GetBlockId() << "\" "; } - output_<< std::endl; + output_<< "\n"; } void PrintExceptionHandlers(HBasicBlock* block) { @@ -272,7 +278,7 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { !disasm_info_->GetSlowPathIntervals().empty()) { output_ << " \"" << kDisassemblyBlockSlowPaths << "\" "; } - output_<< std::endl; + output_<< "\n"; } void DumpLocation(std::ostream& stream, const Location& location) { @@ -367,6 +373,10 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { << load_class->NeedsAccessCheck() << std::noboolalpha; } + void VisitLoadString(HLoadString* load_string) OVERRIDE { + StartAttributeStream("load_kind") << load_string->GetLoadKind(); + } + void VisitCheckCast(HCheckCast* check_cast) OVERRIDE { StartAttributeStream("check_kind") << check_cast->GetTypeCheckKind(); StartAttributeStream("must_do_null_check") << std::boolalpha @@ -418,6 +428,20 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { StartAttributeStream("intrinsic") << invoke->GetIntrinsic(); } + void VisitInstanceFieldGet(HInstanceFieldGet* iget) OVERRIDE { + StartAttributeStream("field_name") << PrettyField(iget->GetFieldInfo().GetFieldIndex(), + iget->GetFieldInfo().GetDexFile(), + /* with type */ false); + StartAttributeStream("field_type") << iget->GetFieldType(); + } + + void VisitInstanceFieldSet(HInstanceFieldSet* iset) OVERRIDE { + StartAttributeStream("field_name") << PrettyField(iset->GetFieldInfo().GetFieldIndex(), + iset->GetFieldInfo().GetDexFile(), + /* with type */ false); + StartAttributeStream("field_type") << iset->GetFieldType(); + } + void VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet* field_access) OVERRIDE { StartAttributeStream("field_type") << field_access->GetFieldType(); } @@ -574,7 +598,7 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { auto it = disasm_info_->GetInstructionIntervals().find(instruction); if (it != disasm_info_->GetInstructionIntervals().end() && it->second.start != it->second.end) { - output_ << std::endl; + output_ << "\n"; disassembler_->Disassemble(output_, it->second.start, it->second.end); } } @@ -594,7 +618,7 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { output_ << bci << " " << num_uses << " " << GetTypeId(instruction->GetType()) << instruction->GetId() << " "; PrintInstruction(instruction); - output_ << " " << kEndInstructionMarker << std::endl; + output_ << " " << kEndInstructionMarker << "\n"; } } @@ -638,10 +662,10 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { output_ << " 0 0 disasm " << kDisassemblyBlockFrameEntry << " "; GeneratedCodeInterval frame_entry = disasm_info_->GetFrameEntryInterval(); if (frame_entry.start != frame_entry.end) { - output_ << std::endl; + output_ << "\n"; disassembler_->Disassemble(output_, frame_entry.start, frame_entry.end); } - output_ << kEndInstructionMarker << std::endl; + output_ << kEndInstructionMarker << "\n"; DumpEndOfDisassemblyBlock(); } @@ -657,9 +681,9 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { GetGraph()->HasExitBlock() ? GetGraph()->GetExitBlock()->GetBlockId() : -1, -1); for (SlowPathCodeInfo info : disasm_info_->GetSlowPathIntervals()) { - output_ << " 0 0 disasm " << info.slow_path->GetDescription() << std::endl; + output_ << " 0 0 disasm " << info.slow_path->GetDescription() << "\n"; disassembler_->Disassemble(output_, info.code_interval.start, info.code_interval.end); - output_ << kEndInstructionMarker << std::endl; + output_ << kEndInstructionMarker << "\n"; } DumpEndOfDisassemblyBlock(); } @@ -680,6 +704,7 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { DumpDisassemblyBlockForSlowPaths(); } EndTag("cfg"); + Flush(); } void VisitBasicBlock(HBasicBlock* block) OVERRIDE { @@ -719,7 +744,7 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { for (HInputIterator inputs(instruction); !inputs.Done(); inputs.Advance()) { output_ << inputs.Current()->GetId() << " "; } - output_ << "]" << std::endl; + output_ << "]\n"; } EndTag("locals"); EndTag("states"); @@ -761,6 +786,7 @@ void HGraphVisualizer::PrintHeader(const char* method_name) const { printer.PrintProperty("method", method_name); printer.PrintTime("date"); printer.EndTag("compilation"); + printer.Flush(); } void HGraphVisualizer::DumpGraph(const char* pass_name, diff --git a/compiler/optimizing/induction_var_analysis.cc b/compiler/optimizing/induction_var_analysis.cc index 266cb10ab3..c06d19dce0 100644 --- a/compiler/optimizing/induction_var_analysis.cc +++ b/compiler/optimizing/induction_var_analysis.cc @@ -141,7 +141,7 @@ void HInductionVarAnalysis::VisitLoop(HLoopInformation* loop) { DCHECK(stack_.empty()); map_.clear(); - // Determine the loop's trip count. + // Determine the loop's trip-count. VisitControl(loop); } @@ -917,6 +917,7 @@ bool HInductionVarAnalysis::InductionEqual(InductionInfo* info1, info1->induction_class == info2->induction_class && info1->operation == info2->operation && info1->fetch == info2->fetch && + info1->type == info2->type && InductionEqual(info1->op_a, info2->op_a) && InductionEqual(info1->op_b, info2->op_b); } diff --git a/compiler/optimizing/induction_var_analysis_test.cc b/compiler/optimizing/induction_var_analysis_test.cc index 0fbb67d0d9..580d24b74b 100644 --- a/compiler/optimizing/induction_var_analysis_test.cc +++ b/compiler/optimizing/induction_var_analysis_test.cc @@ -157,6 +157,13 @@ class InductionVarAnalysisTest : public CommonCompilerTest { iva_->LookupInfo(loop_body_[d]->GetLoopInformation(), instruction)); } + // Returns true if instructions have identical induction. + bool HaveSameInduction(HInstruction* instruction1, HInstruction* instruction2) { + return HInductionVarAnalysis::InductionEqual( + iva_->LookupInfo(loop_body_[0]->GetLoopInformation(), instruction1), + iva_->LookupInfo(loop_body_[0]->GetLoopInformation(), instruction2)); + } + // Performs InductionVarAnalysis (after proper set up). void PerformInductionVarAnalysis() { graph_->BuildDominatorTree(); @@ -228,6 +235,9 @@ TEST_F(InductionVarAnalysisTest, FindBasicInduction) { EXPECT_STREQ("((1) * i + (0)):PrimInt", GetInductionInfo(store->InputAt(1), 0).c_str()); EXPECT_STREQ("((1) * i + (1)):PrimInt", GetInductionInfo(increment_[0], 0).c_str()); + // Offset matters! + EXPECT_FALSE(HaveSameInduction(store->InputAt(1), increment_[0])); + // Trip-count. EXPECT_STREQ("((100) (TC-loop) ((0) < (100)))", GetInductionInfo(loop_header_[0]->GetLastInstruction(), 0).c_str()); @@ -320,6 +330,10 @@ TEST_F(InductionVarAnalysisTest, FindTwoWayBasicInduction) { PerformInductionVarAnalysis(); EXPECT_STREQ("((1) * i + (1)):PrimInt", GetInductionInfo(store->InputAt(1), 0).c_str()); + + // Both increments get same induction. + EXPECT_TRUE(HaveSameInduction(store->InputAt(1), inc1)); + EXPECT_TRUE(HaveSameInduction(store->InputAt(1), inc2)); } TEST_F(InductionVarAnalysisTest, FindTwoWayDerivedInduction) { @@ -570,6 +584,33 @@ TEST_F(InductionVarAnalysisTest, FindDeepLoopInduction) { } } +TEST_F(InductionVarAnalysisTest, ByteInductionIntLoopControl) { + // Setup: + // for (int i = 0; i < 100; i++) { + // k = (byte) i; + // a[k] = 0; + // a[i] = 0; + // } + BuildLoopNest(1); + HInstruction *conv = InsertInstruction( + new (&allocator_) HTypeConversion(Primitive::kPrimByte, basic_[0], -1), 0); + HInstruction* store1 = InsertArrayStore(conv, 0); + HInstruction* store2 = InsertArrayStore(basic_[0], 0); + PerformInductionVarAnalysis(); + + // Regular int induction (i) is "transferred" over conversion into byte induction (k). + EXPECT_STREQ("((1) * i + (0)):PrimByte", GetInductionInfo(store1->InputAt(1), 0).c_str()); + EXPECT_STREQ("((1) * i + (0)):PrimInt", GetInductionInfo(store2->InputAt(1), 0).c_str()); + EXPECT_STREQ("((1) * i + (1)):PrimInt", GetInductionInfo(increment_[0], 0).c_str()); + + // Type matters! + EXPECT_FALSE(HaveSameInduction(store1->InputAt(1), store2->InputAt(1))); + + // Trip-count. + EXPECT_STREQ("((100) (TC-loop) ((0) < (100)))", + GetInductionInfo(loop_header_[0]->GetLastInstruction(), 0).c_str()); +} + TEST_F(InductionVarAnalysisTest, ByteLoopControl1) { // Setup: // for (byte i = -128; i < 127; i++) { // just fits! diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 440a2821c1..fce9ab8b85 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -920,8 +920,7 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, bool needs_constructor_barrier = false; for (size_t i = 0; i != number_of_iputs; ++i) { HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, iput_args[i]); - if (!value->IsConstant() || - (!value->AsConstant()->IsZero() && !value->IsNullConstant())) { + if (!value->IsConstant() || !value->AsConstant()->IsZeroBitPattern()) { if (dex_cache.GetReference() == nullptr) { dex_cache = handles_->NewHandle(resolved_method->GetDexCache()); } @@ -1013,7 +1012,6 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); Handle<mirror::DexCache> dex_cache(handles_->NewHandle(resolved_method->GetDexCache())); DexCompilationUnit dex_compilation_unit( - nullptr, caller_compilation_unit_.GetClassLoader(), class_linker, callee_dex_file, diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index 820c696033..1249b48e1e 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -56,7 +56,6 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { bool TryDeMorganNegationFactoring(HBinaryOperation* op); void VisitShift(HBinaryOperation* shift); - void VisitSuspendCheck(HSuspendCheck* check) OVERRIDE; void VisitEqual(HEqual* equal) OVERRIDE; void VisitNotEqual(HNotEqual* equal) OVERRIDE; void VisitBooleanNot(HBooleanNot* bool_not) OVERRIDE; @@ -240,8 +239,9 @@ void InstructionSimplifierVisitor::VisitShift(HBinaryOperation* instruction) { if (input_cst != nullptr) { int64_t cst = Int64FromConstant(input_cst); - int64_t mask = - (input_other->GetType() == Primitive::kPrimLong) ? kMaxLongShiftValue : kMaxIntShiftValue; + int64_t mask = (input_other->GetType() == Primitive::kPrimLong) + ? kMaxLongShiftDistance + : kMaxIntShiftDistance; if ((cst & mask) == 0) { // Replace code looking like // SHL dst, src, 0 @@ -554,22 +554,6 @@ void InstructionSimplifierVisitor::VisitStaticFieldSet(HStaticFieldSet* instruct } } -void InstructionSimplifierVisitor::VisitSuspendCheck(HSuspendCheck* check) { - HBasicBlock* block = check->GetBlock(); - // Currently always keep the suspend check at entry. - if (block->IsEntryBlock()) return; - - // Currently always keep suspend checks at loop entry. - if (block->IsLoopHeader() && block->GetFirstInstruction() == check) { - DCHECK(block->GetLoopInformation()->GetSuspendCheck() == check); - return; - } - - // Remove the suspend check that was added at build time for the baseline - // compiler. - block->RemoveInstruction(check); -} - static HCondition* GetOppositeConditionSwapOps(ArenaAllocator* arena, HInstruction* cond) { HInstruction *lhs = cond->InputAt(0); HInstruction *rhs = cond->InputAt(1); @@ -608,12 +592,12 @@ void InstructionSimplifierVisitor::VisitEqual(HEqual* equal) { HBasicBlock* block = equal->GetBlock(); // We are comparing the boolean to a constant which is of type int and can // be any constant. - if (input_const->AsIntConstant()->IsOne()) { + if (input_const->AsIntConstant()->IsTrue()) { // Replace (bool_value == true) with bool_value equal->ReplaceWith(input_value); block->RemoveInstruction(equal); RecordSimplification(); - } else if (input_const->AsIntConstant()->IsZero()) { + } else if (input_const->AsIntConstant()->IsFalse()) { equal->ReplaceWith(GetGraph()->InsertOppositeCondition(input_value, equal)); block->RemoveInstruction(equal); RecordSimplification(); @@ -639,11 +623,11 @@ void InstructionSimplifierVisitor::VisitNotEqual(HNotEqual* not_equal) { HBasicBlock* block = not_equal->GetBlock(); // We are comparing the boolean to a constant which is of type int and can // be any constant. - if (input_const->AsIntConstant()->IsOne()) { + if (input_const->AsIntConstant()->IsTrue()) { not_equal->ReplaceWith(GetGraph()->InsertOppositeCondition(input_value, not_equal)); block->RemoveInstruction(not_equal); RecordSimplification(); - } else if (input_const->AsIntConstant()->IsZero()) { + } else if (input_const->AsIntConstant()->IsFalse()) { // Replace (bool_value != false) with bool_value not_equal->ReplaceWith(input_value); block->RemoveInstruction(not_equal); @@ -668,10 +652,10 @@ void InstructionSimplifierVisitor::VisitBooleanNot(HBooleanNot* bool_not) { if (input->IsIntConstant()) { // Replace !(true/false) with false/true. - if (input->AsIntConstant()->IsOne()) { + if (input->AsIntConstant()->IsTrue()) { replace_with = GetGraph()->GetIntConstant(0); } else { - DCHECK(input->AsIntConstant()->IsZero()); + DCHECK(input->AsIntConstant()->IsFalse()) << input->AsIntConstant()->GetValue(); replace_with = GetGraph()->GetIntConstant(1); } } else if (input->IsBooleanNot()) { @@ -712,19 +696,19 @@ void InstructionSimplifierVisitor::VisitSelect(HSelect* select) { // Replace (cond ? x : x) with (x). replace_with = true_value; } else if (condition->IsIntConstant()) { - if (condition->AsIntConstant()->IsOne()) { + if (condition->AsIntConstant()->IsTrue()) { // Replace (true ? x : y) with (x). replace_with = true_value; } else { // Replace (false ? x : y) with (y). - DCHECK(condition->AsIntConstant()->IsZero()); + DCHECK(condition->AsIntConstant()->IsFalse()) << condition->AsIntConstant()->GetValue(); replace_with = false_value; } } else if (true_value->IsIntConstant() && false_value->IsIntConstant()) { - if (true_value->AsIntConstant()->IsOne() && false_value->AsIntConstant()->IsZero()) { + if (true_value->AsIntConstant()->IsTrue() && false_value->AsIntConstant()->IsFalse()) { // Replace (cond ? true : false) with (cond). replace_with = condition; - } else if (true_value->AsIntConstant()->IsZero() && false_value->AsIntConstant()->IsOne()) { + } else if (true_value->AsIntConstant()->IsFalse() && false_value->AsIntConstant()->IsTrue()) { // Replace (cond ? false : true) with (!cond). replace_with = GetGraph()->InsertOppositeCondition(condition, select); } @@ -904,7 +888,7 @@ void InstructionSimplifierVisitor::VisitTypeConversion(HTypeConversion* instruct void InstructionSimplifierVisitor::VisitAdd(HAdd* instruction) { HConstant* input_cst = instruction->GetConstantRight(); HInstruction* input_other = instruction->GetLeastConstantLeft(); - if ((input_cst != nullptr) && input_cst->IsZero()) { + if ((input_cst != nullptr) && input_cst->IsArithmeticZero()) { // Replace code looking like // ADD dst, src, 0 // with @@ -1335,7 +1319,7 @@ void InstructionSimplifierVisitor::VisitOr(HOr* instruction) { HConstant* input_cst = instruction->GetConstantRight(); HInstruction* input_other = instruction->GetLeastConstantLeft(); - if ((input_cst != nullptr) && input_cst->IsZero()) { + if ((input_cst != nullptr) && input_cst->IsZeroBitPattern()) { // Replace code looking like // OR dst, src, 0 // with @@ -1380,7 +1364,7 @@ void InstructionSimplifierVisitor::VisitSub(HSub* instruction) { return; } - if ((input_cst != nullptr) && input_cst->IsZero()) { + if ((input_cst != nullptr) && input_cst->IsArithmeticZero()) { // Replace code looking like // SUB dst, src, 0 // with @@ -1461,7 +1445,7 @@ void InstructionSimplifierVisitor::VisitXor(HXor* instruction) { HConstant* input_cst = instruction->GetConstantRight(); HInstruction* input_other = instruction->GetLeastConstantLeft(); - if ((input_cst != nullptr) && input_cst->IsZero()) { + if ((input_cst != nullptr) && input_cst->IsZeroBitPattern()) { // Replace code looking like // XOR dst, src, 0 // with @@ -1745,7 +1729,7 @@ void InstructionSimplifierVisitor::VisitInvoke(HInvoke* instruction) { void InstructionSimplifierVisitor::VisitDeoptimize(HDeoptimize* deoptimize) { HInstruction* cond = deoptimize->InputAt(0); if (cond->IsConstant()) { - if (cond->AsIntConstant()->IsZero()) { + if (cond->AsIntConstant()->IsFalse()) { // Never deopt: instruction can be removed. deoptimize->GetBlock()->RemoveInstruction(deoptimize); } else { diff --git a/compiler/optimizing/instruction_simplifier_arm64.h b/compiler/optimizing/instruction_simplifier_arm64.h index 338120bbbc..da269980e8 100644 --- a/compiler/optimizing/instruction_simplifier_arm64.h +++ b/compiler/optimizing/instruction_simplifier_arm64.h @@ -51,6 +51,22 @@ class InstructionSimplifierArm64Visitor : public HGraphVisitor { return TryMergeIntoShifterOperand(use, bitfield_op, true); } + /** + * This simplifier uses a special-purpose BB visitor. + * (1) No need to visit Phi nodes. + * (2) Since statements can be removed in a "forward" fashion, + * the visitor should test if each statement is still there. + */ + void VisitBasicBlock(HBasicBlock* block) OVERRIDE { + // TODO: fragile iteration, provide more robust iterators? + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + HInstruction* instruction = it.Current(); + if (instruction->IsInBlock()) { + instruction->Accept(this); + } + } + } + // HInstruction visitors, sorted alphabetically. void VisitAnd(HAnd* instruction) OVERRIDE; void VisitArrayGet(HArrayGet* instruction) OVERRIDE; diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index c83340b1f6..950448136d 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -86,11 +86,7 @@ void HGraph::FindBackEdges(ArenaBitVector* visited) { } } -static void RemoveAsUser(HInstruction* instruction) { - for (size_t i = 0; i < instruction->InputCount(); i++) { - instruction->RemoveAsUserOfInput(i); - } - +static void RemoveEnvironmentUses(HInstruction* instruction) { for (HEnvironment* environment = instruction->GetEnvironment(); environment != nullptr; environment = environment->GetParent()) { @@ -102,6 +98,14 @@ static void RemoveAsUser(HInstruction* instruction) { } } +static void RemoveAsUser(HInstruction* instruction) { + for (size_t i = 0; i < instruction->InputCount(); i++) { + instruction->RemoveAsUserOfInput(i); + } + + RemoveEnvironmentUses(instruction); +} + void HGraph::RemoveInstructionsAsUsersFromDeadBlocks(const ArenaBitVector& visited) const { for (size_t i = 0; i < blocks_.size(); ++i) { if (!visited.IsBitSet(i)) { @@ -162,7 +166,7 @@ GraphAnalysisResult HGraph::BuildDominatorTree() { // (6) Compute the dominance information and the reverse post order. ComputeDominanceInformation(); - // (7) Analyze loops discover through back edge analysis, and + // (7) Analyze loops discovered through back edge analysis, and // set the loop information on each block. GraphAnalysisResult result = AnalyzeLoops(); if (result != kAnalysisSuccess) { @@ -247,7 +251,7 @@ void HGraph::ComputeDominanceInformation() { } // Populate `dominated_blocks_` information after computing all dominators. - // The potential presence of irreducible loops require to do it after. + // The potential presence of irreducible loops requires to do it after. for (HReversePostOrderIterator it(*this); !it.Done(); it.Advance()) { HBasicBlock* block = it.Current(); if (!block->IsEntryBlock()) { @@ -460,7 +464,7 @@ void HGraph::SimplifyCFG() { if (block->IsLoopHeader()) { SimplifyLoop(block); } else if (!block->IsEntryBlock() && block->GetFirstInstruction()->IsSuspendCheck()) { - // We are being called by the dead code elimiation pass, and what used to be + // We are being called by the dead code elimination pass, and what used to be // a loop got dismantled. Just remove the suspend check. block->RemoveInstruction(block->GetFirstInstruction()); } @@ -1007,6 +1011,11 @@ bool HInstruction::StrictlyDominates(HInstruction* other_instruction) const { } } +void HInstruction::RemoveEnvironment() { + RemoveEnvironmentUses(this); + environment_ = nullptr; +} + void HInstruction::ReplaceWith(HInstruction* other) { DCHECK(other != nullptr); for (HUseIterator<HInstruction*> it(GetUses()); !it.Done(); it.Advance()) { @@ -1654,21 +1663,77 @@ void HBasicBlock::DisconnectAndDelete() { // iteration. DCHECK(dominated_blocks_.empty()); - // (1) Remove the block from all loops it is included in. - for (HLoopInformationOutwardIterator it(*this); !it.Done(); it.Advance()) { - HLoopInformation* loop_info = it.Current(); - loop_info->Remove(this); - if (loop_info->IsBackEdge(*this)) { - // If this was the last back edge of the loop, we deliberately leave the - // loop in an inconsistent state and will fail GraphChecker unless the - // entire loop is removed during the pass. - loop_info->RemoveBackEdge(this); + // The following steps gradually remove the block from all its dependants in + // post order (b/27683071). + + // (1) Store a basic block that we'll use in step (5) to find loops to be updated. + // We need to do this before step (4) which destroys the predecessor list. + HBasicBlock* loop_update_start = this; + if (IsLoopHeader()) { + HLoopInformation* loop_info = GetLoopInformation(); + // All other blocks in this loop should have been removed because the header + // was their dominator. + // Note that we do not remove `this` from `loop_info` as it is unreachable. + DCHECK(!loop_info->IsIrreducible()); + DCHECK_EQ(loop_info->GetBlocks().NumSetBits(), 1u); + DCHECK_EQ(static_cast<uint32_t>(loop_info->GetBlocks().GetHighestBitSet()), GetBlockId()); + loop_update_start = loop_info->GetPreHeader(); + } + + // (2) Disconnect the block from its successors and update their phis. + for (HBasicBlock* successor : successors_) { + // Delete this block from the list of predecessors. + size_t this_index = successor->GetPredecessorIndexOf(this); + successor->predecessors_.erase(successor->predecessors_.begin() + this_index); + + // Check that `successor` has other predecessors, otherwise `this` is the + // dominator of `successor` which violates the order DCHECKed at the top. + DCHECK(!successor->predecessors_.empty()); + + // Remove this block's entries in the successor's phis. Skip exceptional + // successors because catch phi inputs do not correspond to predecessor + // blocks but throwing instructions. The inputs of the catch phis will be + // updated in step (3). + if (!successor->IsCatchBlock()) { + if (successor->predecessors_.size() == 1u) { + // The successor has just one predecessor left. Replace phis with the only + // remaining input. + for (HInstructionIterator phi_it(successor->GetPhis()); !phi_it.Done(); phi_it.Advance()) { + HPhi* phi = phi_it.Current()->AsPhi(); + phi->ReplaceWith(phi->InputAt(1 - this_index)); + successor->RemovePhi(phi); + } + } else { + for (HInstructionIterator phi_it(successor->GetPhis()); !phi_it.Done(); phi_it.Advance()) { + phi_it.Current()->AsPhi()->RemoveInputAt(this_index); + } + } } } + successors_.clear(); - // (2) Disconnect the block from its predecessors and update their + // (3) Remove instructions and phis. Instructions should have no remaining uses + // except in catch phis. If an instruction is used by a catch phi at `index`, + // remove `index`-th input of all phis in the catch block since they are + // guaranteed dead. Note that we may miss dead inputs this way but the + // graph will always remain consistent. + for (HBackwardInstructionIterator it(GetInstructions()); !it.Done(); it.Advance()) { + HInstruction* insn = it.Current(); + RemoveUsesOfDeadInstruction(insn); + RemoveInstruction(insn); + } + for (HInstructionIterator it(GetPhis()); !it.Done(); it.Advance()) { + HPhi* insn = it.Current()->AsPhi(); + RemoveUsesOfDeadInstruction(insn); + RemovePhi(insn); + } + + // (4) Disconnect the block from its predecessors and update their // control-flow instructions. for (HBasicBlock* predecessor : predecessors_) { + // We should not see any back edges as they would have been removed by step (3). + DCHECK(!IsInLoop() || !GetLoopInformation()->IsBackEdge(*predecessor)); + HInstruction* last_instruction = predecessor->GetLastInstruction(); if (last_instruction->IsTryBoundary() && !IsCatchBlock()) { // This block is the only normal-flow successor of the TryBoundary which @@ -1712,58 +1777,25 @@ void HBasicBlock::DisconnectAndDelete() { } predecessors_.clear(); - // (3) Disconnect the block from its successors and update their phis. - for (HBasicBlock* successor : successors_) { - // Delete this block from the list of predecessors. - size_t this_index = successor->GetPredecessorIndexOf(this); - successor->predecessors_.erase(successor->predecessors_.begin() + this_index); - - // Check that `successor` has other predecessors, otherwise `this` is the - // dominator of `successor` which violates the order DCHECKed at the top. - DCHECK(!successor->predecessors_.empty()); - - // Remove this block's entries in the successor's phis. Skip exceptional - // successors because catch phi inputs do not correspond to predecessor - // blocks but throwing instructions. Their inputs will be updated in step (4). - if (!successor->IsCatchBlock()) { - if (successor->predecessors_.size() == 1u) { - // The successor has just one predecessor left. Replace phis with the only - // remaining input. - for (HInstructionIterator phi_it(successor->GetPhis()); !phi_it.Done(); phi_it.Advance()) { - HPhi* phi = phi_it.Current()->AsPhi(); - phi->ReplaceWith(phi->InputAt(1 - this_index)); - successor->RemovePhi(phi); - } - } else { - for (HInstructionIterator phi_it(successor->GetPhis()); !phi_it.Done(); phi_it.Advance()) { - phi_it.Current()->AsPhi()->RemoveInputAt(this_index); - } - } + // (5) Remove the block from all loops it is included in. Skip the inner-most + // loop if this is the loop header (see definition of `loop_update_start`) + // because the loop header's predecessor list has been destroyed in step (4). + for (HLoopInformationOutwardIterator it(*loop_update_start); !it.Done(); it.Advance()) { + HLoopInformation* loop_info = it.Current(); + loop_info->Remove(this); + if (loop_info->IsBackEdge(*this)) { + // If this was the last back edge of the loop, we deliberately leave the + // loop in an inconsistent state and will fail GraphChecker unless the + // entire loop is removed during the pass. + loop_info->RemoveBackEdge(this); } } - successors_.clear(); - // (4) Remove instructions and phis. Instructions should have no remaining uses - // except in catch phis. If an instruction is used by a catch phi at `index`, - // remove `index`-th input of all phis in the catch block since they are - // guaranteed dead. Note that we may miss dead inputs this way but the - // graph will always remain consistent. - for (HBackwardInstructionIterator it(GetInstructions()); !it.Done(); it.Advance()) { - HInstruction* insn = it.Current(); - RemoveUsesOfDeadInstruction(insn); - RemoveInstruction(insn); - } - for (HInstructionIterator it(GetPhis()); !it.Done(); it.Advance()) { - HPhi* insn = it.Current()->AsPhi(); - RemoveUsesOfDeadInstruction(insn); - RemovePhi(insn); - } - - // Disconnect from the dominator. + // (6) Disconnect from the dominator. dominator_->RemoveDominatedBlock(this); SetDominator(nullptr); - // Delete from the graph, update reverse post order. + // (7) Delete from the graph, update reverse post order. graph_->DeleteDeadEmptyBlock(this); SetGraph(nullptr); } @@ -2364,6 +2396,59 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::ClinitCheckReq } } +bool HLoadString::InstructionDataEquals(HInstruction* other) const { + HLoadString* other_load_string = other->AsLoadString(); + if (string_index_ != other_load_string->string_index_ || + GetPackedFields() != other_load_string->GetPackedFields()) { + return false; + } + LoadKind load_kind = GetLoadKind(); + if (HasAddress(load_kind)) { + return GetAddress() == other_load_string->GetAddress(); + } else if (HasStringReference(load_kind)) { + return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile()); + } else { + DCHECK(HasDexCacheReference(load_kind)) << load_kind; + // If the string indexes and dex files are the same, dex cache element offsets + // must also be the same, so we don't need to compare them. + return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile()); + } +} + +void HLoadString::SetLoadKindInternal(LoadKind load_kind) { + // Once sharpened, the load kind should not be changed again. + DCHECK_EQ(GetLoadKind(), LoadKind::kDexCacheViaMethod); + SetPackedField<LoadKindField>(load_kind); + + if (load_kind != LoadKind::kDexCacheViaMethod) { + RemoveAsUserOfInput(0u); + SetRawInputAt(0u, nullptr); + } + if (!NeedsEnvironment()) { + RemoveEnvironment(); + } +} + +std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs) { + switch (rhs) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + return os << "BootImageLinkTimeAddress"; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + return os << "BootImageLinkTimePcRelative"; + case HLoadString::LoadKind::kBootImageAddress: + return os << "BootImageAddress"; + case HLoadString::LoadKind::kDexCacheAddress: + return os << "DexCacheAddress"; + case HLoadString::LoadKind::kDexCachePcRelative: + return os << "DexCachePcRelative"; + case HLoadString::LoadKind::kDexCacheViaMethod: + return os << "DexCacheViaMethod"; + default: + LOG(FATAL) << "Unknown HLoadString::LoadKind: " << static_cast<int>(rhs); + UNREACHABLE(); + } +} + void HInstruction::RemoveEnvironmentUsers() { for (HUseIterator<HEnvironment*> use_it(GetEnvUses()); !use_it.Done(); use_it.Advance()) { HUseListNode<HEnvironment*>* user_node = use_it.Current(); @@ -2373,7 +2458,7 @@ void HInstruction::RemoveEnvironmentUsers() { env_uses_.Clear(); } -// Returns an instruction with the opposite boolean value from 'cond'. +// Returns an instruction with the opposite Boolean value from 'cond'. HInstruction* HGraph::InsertOppositeCondition(HInstruction* cond, HInstruction* cursor) { ArenaAllocator* allocator = GetArena(); @@ -2402,10 +2487,10 @@ HInstruction* HGraph::InsertOppositeCondition(HInstruction* cond, HInstruction* return replacement; } else if (cond->IsIntConstant()) { HIntConstant* int_const = cond->AsIntConstant(); - if (int_const->IsZero()) { + if (int_const->IsFalse()) { return GetIntConstant(1); } else { - DCHECK(int_const->IsOne()); + DCHECK(int_const->IsTrue()) << int_const->GetValue(); return GetIntConstant(0); } } else { diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index a09ad00cdd..ba4242112b 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -72,8 +72,10 @@ static const int kDefaultNumberOfExceptionalPredecessors = 0; static const int kDefaultNumberOfDominatedBlocks = 1; static const int kDefaultNumberOfBackEdges = 1; -static constexpr uint32_t kMaxIntShiftValue = 0x1f; -static constexpr uint64_t kMaxLongShiftValue = 0x3f; +// The maximum (meaningful) distance (31) that can be used in an integer shift/rotate operation. +static constexpr int32_t kMaxIntShiftDistance = 0x1f; +// The maximum (meaningful) distance (63) that can be used in a long shift/rotate operation. +static constexpr int32_t kMaxLongShiftDistance = 0x3f; static constexpr uint32_t kUnknownFieldIndex = static_cast<uint32_t>(-1); static constexpr uint16_t kUnknownClassDefIndex = static_cast<uint16_t>(-1); @@ -1993,6 +1995,8 @@ class HInstruction : public ArenaObject<kArenaAllocInstruction> { environment_ = environment; } + void RemoveEnvironment(); + // Set the environment of this instruction, copying it from `environment`. While // copying, the uses lists are being updated. void CopyEnvironmentFrom(HEnvironment* environment) { @@ -2427,8 +2431,13 @@ class HConstant : public HExpression<0> { bool CanBeMoved() const OVERRIDE { return true; } + // Is this constant -1 in the arithmetic sense? virtual bool IsMinusOne() const { return false; } - virtual bool IsZero() const { return false; } + // Is this constant 0 in the arithmetic sense? + virtual bool IsArithmeticZero() const { return false; } + // Is this constant a 0-bit pattern? + virtual bool IsZeroBitPattern() const { return false; } + // Is this constant 1 in the arithmetic sense? virtual bool IsOne() const { return false; } virtual uint64_t GetValueAsUint64() const = 0; @@ -2449,6 +2458,9 @@ class HNullConstant : public HConstant { size_t ComputeHashCode() const OVERRIDE { return 0; } + // The null constant representation is a 0-bit pattern. + virtual bool IsZeroBitPattern() const { return true; } + DECLARE_INSTRUCTION(NullConstant); private: @@ -2476,9 +2488,15 @@ class HIntConstant : public HConstant { size_t ComputeHashCode() const OVERRIDE { return GetValue(); } bool IsMinusOne() const OVERRIDE { return GetValue() == -1; } - bool IsZero() const OVERRIDE { return GetValue() == 0; } + bool IsArithmeticZero() const OVERRIDE { return GetValue() == 0; } + bool IsZeroBitPattern() const OVERRIDE { return GetValue() == 0; } bool IsOne() const OVERRIDE { return GetValue() == 1; } + // Integer constants are used to encode Boolean values as well, + // where 1 means true and 0 means false. + bool IsTrue() const { return GetValue() == 1; } + bool IsFalse() const { return GetValue() == 0; } + DECLARE_INSTRUCTION(IntConstant); private: @@ -2509,7 +2527,8 @@ class HLongConstant : public HConstant { size_t ComputeHashCode() const OVERRIDE { return static_cast<size_t>(GetValue()); } bool IsMinusOne() const OVERRIDE { return GetValue() == -1; } - bool IsZero() const OVERRIDE { return GetValue() == 0; } + bool IsArithmeticZero() const OVERRIDE { return GetValue() == 0; } + bool IsZeroBitPattern() const OVERRIDE { return GetValue() == 0; } bool IsOne() const OVERRIDE { return GetValue() == 1; } DECLARE_INSTRUCTION(LongConstant); @@ -2542,7 +2561,16 @@ class HFloatConstant : public HConstant { bool IsMinusOne() const OVERRIDE { return bit_cast<uint32_t, float>(value_) == bit_cast<uint32_t, float>((-1.0f)); } - bool IsZero() const OVERRIDE { + bool IsArithmeticZero() const OVERRIDE { + return std::fpclassify(value_) == FP_ZERO; + } + bool IsArithmeticPositiveZero() const { + return IsArithmeticZero() && !std::signbit(value_); + } + bool IsArithmeticNegativeZero() const { + return IsArithmeticZero() && std::signbit(value_); + } + bool IsZeroBitPattern() const OVERRIDE { return bit_cast<uint32_t, float>(value_) == bit_cast<uint32_t, float>(0.0f); } bool IsOne() const OVERRIDE { @@ -2584,7 +2612,16 @@ class HDoubleConstant : public HConstant { bool IsMinusOne() const OVERRIDE { return bit_cast<uint64_t, double>(value_) == bit_cast<uint64_t, double>((-1.0)); } - bool IsZero() const OVERRIDE { + bool IsArithmeticZero() const OVERRIDE { + return std::fpclassify(value_) == FP_ZERO; + } + bool IsArithmeticPositiveZero() const { + return IsArithmeticZero() && !std::signbit(value_); + } + bool IsArithmeticNegativeZero() const { + return IsArithmeticZero() && std::signbit(value_); + } + bool IsZeroBitPattern() const OVERRIDE { return bit_cast<uint64_t, double>(value_) == bit_cast<uint64_t, double>((0.0)); } bool IsOne() const OVERRIDE { @@ -3442,10 +3479,8 @@ class HCompare : public HBinaryOperation { SideEffectsForArchRuntimeCalls(comparison_type), dex_pc) { SetPackedField<ComparisonBiasField>(bias); - if (kIsDebugBuild) { - DCHECK_EQ(comparison_type, Primitive::PrimitiveKind(first->GetType())); - DCHECK_EQ(comparison_type, Primitive::PrimitiveKind(second->GetType())); - } + DCHECK_EQ(comparison_type, Primitive::PrimitiveKind(first->GetType())); + DCHECK_EQ(comparison_type, Primitive::PrimitiveKind(second->GetType())); } template <typename T> @@ -4447,37 +4482,39 @@ class HDivZeroCheck : public HExpression<1> { class HShl : public HBinaryOperation { public: HShl(Primitive::Type result_type, - HInstruction* left, - HInstruction* right, + HInstruction* value, + HInstruction* distance, uint32_t dex_pc = kNoDexPc) - : HBinaryOperation(result_type, left, right, SideEffects::None(), dex_pc) {} + : HBinaryOperation(result_type, value, distance, SideEffects::None(), dex_pc) { + DCHECK_EQ(result_type, Primitive::PrimitiveKind(value->GetType())); + DCHECK_EQ(Primitive::kPrimInt, Primitive::PrimitiveKind(distance->GetType())); + } - template <typename T, typename U, typename V> - T Compute(T x, U y, V max_shift_value) const { - static_assert(std::is_same<V, typename std::make_unsigned<T>::type>::value, - "V is not the unsigned integer type corresponding to T"); - return x << (y & max_shift_value); + template <typename T> + T Compute(T value, int32_t distance, int32_t max_shift_distance) const { + return value << (distance & max_shift_distance); } - HConstant* Evaluate(HIntConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HIntConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetIntConstant( - Compute(x->GetValue(), y->GetValue(), kMaxIntShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxIntShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HLongConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxLongShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HLongConstant* y) const OVERRIDE { - return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + HConstant* Evaluate(HLongConstant* value ATTRIBUTE_UNUSED, + HLongConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { + LOG(FATAL) << DebugName() << " is not defined for the (long, long) case."; + UNREACHABLE(); } - HConstant* Evaluate(HFloatConstant* x ATTRIBUTE_UNUSED, - HFloatConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HFloatConstant* value ATTRIBUTE_UNUSED, + HFloatConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for float values"; UNREACHABLE(); } - HConstant* Evaluate(HDoubleConstant* x ATTRIBUTE_UNUSED, - HDoubleConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HDoubleConstant* value ATTRIBUTE_UNUSED, + HDoubleConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for double values"; UNREACHABLE(); } @@ -4491,37 +4528,39 @@ class HShl : public HBinaryOperation { class HShr : public HBinaryOperation { public: HShr(Primitive::Type result_type, - HInstruction* left, - HInstruction* right, + HInstruction* value, + HInstruction* distance, uint32_t dex_pc = kNoDexPc) - : HBinaryOperation(result_type, left, right, SideEffects::None(), dex_pc) {} + : HBinaryOperation(result_type, value, distance, SideEffects::None(), dex_pc) { + DCHECK_EQ(result_type, Primitive::PrimitiveKind(value->GetType())); + DCHECK_EQ(Primitive::kPrimInt, Primitive::PrimitiveKind(distance->GetType())); + } - template <typename T, typename U, typename V> - T Compute(T x, U y, V max_shift_value) const { - static_assert(std::is_same<V, typename std::make_unsigned<T>::type>::value, - "V is not the unsigned integer type corresponding to T"); - return x >> (y & max_shift_value); + template <typename T> + T Compute(T value, int32_t distance, int32_t max_shift_distance) const { + return value >> (distance & max_shift_distance); } - HConstant* Evaluate(HIntConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HIntConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetIntConstant( - Compute(x->GetValue(), y->GetValue(), kMaxIntShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxIntShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HLongConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxLongShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HLongConstant* y) const OVERRIDE { - return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + HConstant* Evaluate(HLongConstant* value ATTRIBUTE_UNUSED, + HLongConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { + LOG(FATAL) << DebugName() << " is not defined for the (long, long) case."; + UNREACHABLE(); } - HConstant* Evaluate(HFloatConstant* x ATTRIBUTE_UNUSED, - HFloatConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HFloatConstant* value ATTRIBUTE_UNUSED, + HFloatConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for float values"; UNREACHABLE(); } - HConstant* Evaluate(HDoubleConstant* x ATTRIBUTE_UNUSED, - HDoubleConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HDoubleConstant* value ATTRIBUTE_UNUSED, + HDoubleConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for double values"; UNREACHABLE(); } @@ -4535,38 +4574,41 @@ class HShr : public HBinaryOperation { class HUShr : public HBinaryOperation { public: HUShr(Primitive::Type result_type, - HInstruction* left, - HInstruction* right, + HInstruction* value, + HInstruction* distance, uint32_t dex_pc = kNoDexPc) - : HBinaryOperation(result_type, left, right, SideEffects::None(), dex_pc) {} + : HBinaryOperation(result_type, value, distance, SideEffects::None(), dex_pc) { + DCHECK_EQ(result_type, Primitive::PrimitiveKind(value->GetType())); + DCHECK_EQ(Primitive::kPrimInt, Primitive::PrimitiveKind(distance->GetType())); + } - template <typename T, typename U, typename V> - T Compute(T x, U y, V max_shift_value) const { - static_assert(std::is_same<V, typename std::make_unsigned<T>::type>::value, - "V is not the unsigned integer type corresponding to T"); - V ux = static_cast<V>(x); - return static_cast<T>(ux >> (y & max_shift_value)); + template <typename T> + T Compute(T value, int32_t distance, int32_t max_shift_distance) const { + typedef typename std::make_unsigned<T>::type V; + V ux = static_cast<V>(value); + return static_cast<T>(ux >> (distance & max_shift_distance)); } - HConstant* Evaluate(HIntConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HIntConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetIntConstant( - Compute(x->GetValue(), y->GetValue(), kMaxIntShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxIntShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HLongConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxLongShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HLongConstant* y) const OVERRIDE { - return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + HConstant* Evaluate(HLongConstant* value ATTRIBUTE_UNUSED, + HLongConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { + LOG(FATAL) << DebugName() << " is not defined for the (long, long) case."; + UNREACHABLE(); } - HConstant* Evaluate(HFloatConstant* x ATTRIBUTE_UNUSED, - HFloatConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HFloatConstant* value ATTRIBUTE_UNUSED, + HFloatConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for float values"; UNREACHABLE(); } - HConstant* Evaluate(HDoubleConstant* x ATTRIBUTE_UNUSED, - HDoubleConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HDoubleConstant* value ATTRIBUTE_UNUSED, + HDoubleConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for double values"; UNREACHABLE(); } @@ -4692,45 +4734,43 @@ class HRor : public HBinaryOperation { public: HRor(Primitive::Type result_type, HInstruction* value, HInstruction* distance) : HBinaryOperation(result_type, value, distance) { - if (kIsDebugBuild) { - DCHECK_EQ(result_type, Primitive::PrimitiveKind(value->GetType())); - DCHECK_EQ(Primitive::kPrimInt, Primitive::PrimitiveKind(distance->GetType())); - } + DCHECK_EQ(result_type, Primitive::PrimitiveKind(value->GetType())); + DCHECK_EQ(Primitive::kPrimInt, Primitive::PrimitiveKind(distance->GetType())); } - template <typename T, typename U, typename V> - T Compute(T x, U y, V max_shift_value) const { - static_assert(std::is_same<V, typename std::make_unsigned<T>::type>::value, - "V is not the unsigned integer type corresponding to T"); - V ux = static_cast<V>(x); - if ((y & max_shift_value) == 0) { + template <typename T> + T Compute(T value, int32_t distance, int32_t max_shift_value) const { + typedef typename std::make_unsigned<T>::type V; + V ux = static_cast<V>(value); + if ((distance & max_shift_value) == 0) { return static_cast<T>(ux); } else { const V reg_bits = sizeof(T) * 8; - return static_cast<T>(ux >> (y & max_shift_value)) | - (x << (reg_bits - (y & max_shift_value))); + return static_cast<T>(ux >> (distance & max_shift_value)) | + (value << (reg_bits - (distance & max_shift_value))); } } - HConstant* Evaluate(HIntConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HIntConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetIntConstant( - Compute(x->GetValue(), y->GetValue(), kMaxIntShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxIntShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HIntConstant* y) const OVERRIDE { + HConstant* Evaluate(HLongConstant* value, HIntConstant* distance) const OVERRIDE { return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + Compute(value->GetValue(), distance->GetValue(), kMaxLongShiftDistance), GetDexPc()); } - HConstant* Evaluate(HLongConstant* x, HLongConstant* y) const OVERRIDE { - return GetBlock()->GetGraph()->GetLongConstant( - Compute(x->GetValue(), y->GetValue(), kMaxLongShiftValue), GetDexPc()); + HConstant* Evaluate(HLongConstant* value ATTRIBUTE_UNUSED, + HLongConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { + LOG(FATAL) << DebugName() << " is not defined for the (long, long) case."; + UNREACHABLE(); } - HConstant* Evaluate(HFloatConstant* x ATTRIBUTE_UNUSED, - HFloatConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HFloatConstant* value ATTRIBUTE_UNUSED, + HFloatConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for float values"; UNREACHABLE(); } - HConstant* Evaluate(HDoubleConstant* x ATTRIBUTE_UNUSED, - HDoubleConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { + HConstant* Evaluate(HDoubleConstant* value ATTRIBUTE_UNUSED, + HDoubleConstant* distance ATTRIBUTE_UNUSED) const OVERRIDE { LOG(FATAL) << DebugName() << " is not defined for double values"; UNREACHABLE(); } @@ -5519,32 +5559,117 @@ class HLoadClass : public HExpression<1> { class HLoadString : public HExpression<1> { public: + // Determines how to load the String. + enum class LoadKind { + // Use boot image String* address that will be known at link time. + // Used for boot image strings referenced by boot image code in non-PIC mode. + kBootImageLinkTimeAddress, + + // Use PC-relative boot image String* address that will be known at link time. + // Used for boot image strings referenced by boot image code in PIC mode. + kBootImageLinkTimePcRelative, + + // Use a known boot image String* address, embedded in the code by the codegen. + // Used for boot image strings referenced by apps in AOT- and JIT-compiled code. + // Note: codegen needs to emit a linker patch if indicated by compiler options' + // GetIncludePatchInformation(). + kBootImageAddress, + + // Load from the resolved strings array at an absolute address. + // Used for strings outside the boot image referenced by JIT-compiled code. + kDexCacheAddress, + + // Load from resolved strings array in the dex cache using a PC-relative load. + // Used for strings outside boot image when we know that we can access + // the dex cache arrays using a PC-relative load. + kDexCachePcRelative, + + // Load from resolved strings array accessed through the class loaded from + // the compiled method's own ArtMethod*. This is the default access type when + // all other types are unavailable. + kDexCacheViaMethod, + + kLast = kDexCacheViaMethod + }; + HLoadString(HCurrentMethod* current_method, uint32_t string_index, - uint32_t dex_pc, - bool is_in_dex_cache) + const DexFile& dex_file, + uint32_t dex_pc) : HExpression(Primitive::kPrimNot, SideEffectsForArchRuntimeCalls(), dex_pc), string_index_(string_index) { - SetPackedFlag<kFlagIsInDexCache>(is_in_dex_cache); + SetPackedFlag<kFlagIsInDexCache>(false); + SetPackedField<LoadKindField>(LoadKind::kDexCacheViaMethod); + load_data_.ref.dex_file = &dex_file; SetRawInputAt(0, current_method); } - bool CanBeMoved() const OVERRIDE { return true; } + void SetLoadKindWithAddress(LoadKind load_kind, uint64_t address) { + DCHECK(HasAddress(load_kind)); + load_data_.address = address; + SetLoadKindInternal(load_kind); + } - bool InstructionDataEquals(HInstruction* other) const OVERRIDE { - return other->AsLoadString()->string_index_ == string_index_; + void SetLoadKindWithStringReference(LoadKind load_kind, + const DexFile& dex_file, + uint32_t string_index) { + DCHECK(HasStringReference(load_kind)); + load_data_.ref.dex_file = &dex_file; + string_index_ = string_index; + SetLoadKindInternal(load_kind); + } + + void SetLoadKindWithDexCacheReference(LoadKind load_kind, + const DexFile& dex_file, + uint32_t element_index) { + DCHECK(HasDexCacheReference(load_kind)); + load_data_.ref.dex_file = &dex_file; + load_data_.ref.dex_cache_element_index = element_index; + SetLoadKindInternal(load_kind); } + LoadKind GetLoadKind() const { + return GetPackedField<LoadKindField>(); + } + + const DexFile& GetDexFile() const; + + uint32_t GetStringIndex() const { + DCHECK(HasStringReference(GetLoadKind()) || /* For slow paths. */ !IsInDexCache()); + return string_index_; + } + + uint32_t GetDexCacheElementOffset() const; + + uint64_t GetAddress() const { + DCHECK(HasAddress(GetLoadKind())); + return load_data_.address; + } + + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(HInstruction* other) const OVERRIDE; + size_t ComputeHashCode() const OVERRIDE { return string_index_; } - uint32_t GetStringIndex() const { return string_index_; } + // Will call the runtime if we need to load the string through + // the dex cache and the string is not guaranteed to be there yet. + bool NeedsEnvironment() const OVERRIDE { + LoadKind load_kind = GetLoadKind(); + if (load_kind == LoadKind::kBootImageLinkTimeAddress || + load_kind == LoadKind::kBootImageLinkTimePcRelative || + load_kind == LoadKind::kBootImageAddress) { + return false; + } + return !IsInDexCache(); + } - // Will call the runtime if the string is not already in the dex cache. - bool NeedsEnvironment() const OVERRIDE { return !IsInDexCache(); } + bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { + return GetLoadKind() == LoadKind::kDexCacheViaMethod; + } - bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { return true; } bool CanBeNull() const OVERRIDE { return false; } - bool CanThrow() const OVERRIDE { return !IsInDexCache(); } + bool CanThrow() const OVERRIDE { return NeedsEnvironment(); } static SideEffects SideEffectsForArchRuntimeCalls() { return SideEffects::CanTriggerGC(); @@ -5552,17 +5677,83 @@ class HLoadString : public HExpression<1> { bool IsInDexCache() const { return GetPackedFlag<kFlagIsInDexCache>(); } + void MarkInDexCache() { + SetPackedFlag<kFlagIsInDexCache>(true); + DCHECK(!NeedsEnvironment()); + RemoveEnvironment(); + } + + size_t InputCount() const OVERRIDE { + return (InputAt(0) != nullptr) ? 1u : 0u; + } + + void AddSpecialInput(HInstruction* special_input); + DECLARE_INSTRUCTION(LoadString); private: static constexpr size_t kFlagIsInDexCache = kNumberOfExpressionPackedBits; - static constexpr size_t kNumberOfLoadStringPackedBits = kFlagIsInDexCache + 1; + static constexpr size_t kFieldLoadKind = kFlagIsInDexCache + 1; + static constexpr size_t kFieldLoadKindSize = + MinimumBitsToStore(static_cast<size_t>(LoadKind::kLast)); + static constexpr size_t kNumberOfLoadStringPackedBits = kFieldLoadKind + kFieldLoadKindSize; static_assert(kNumberOfLoadStringPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields."); + using LoadKindField = BitField<LoadKind, kFieldLoadKind, kFieldLoadKindSize>; + + static bool HasStringReference(LoadKind load_kind) { + return load_kind == LoadKind::kBootImageLinkTimeAddress || + load_kind == LoadKind::kBootImageLinkTimePcRelative || + load_kind == LoadKind::kDexCacheViaMethod; + } + + static bool HasAddress(LoadKind load_kind) { + return load_kind == LoadKind::kBootImageAddress || load_kind == LoadKind::kDexCacheAddress; + } + + static bool HasDexCacheReference(LoadKind load_kind) { + return load_kind == LoadKind::kDexCachePcRelative; + } + + void SetLoadKindInternal(LoadKind load_kind); + + // String index serves also as the hash code and it's also needed for slow-paths, + // so it must not be overwritten with other load data. + uint32_t string_index_; - const uint32_t string_index_; + union { + struct { + const DexFile* dex_file; // For string reference and dex cache reference. + uint32_t dex_cache_element_index; // Only for dex cache reference. + } ref; + uint64_t address; // Up to 64-bit, needed for kDexCacheAddress on 64-bit targets. + } load_data_; DISALLOW_COPY_AND_ASSIGN(HLoadString); }; +std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs); + +// Note: defined outside class to see operator<<(., HLoadString::LoadKind). +inline const DexFile& HLoadString::GetDexFile() const { + DCHECK(HasStringReference(GetLoadKind()) || HasDexCacheReference(GetLoadKind())) + << GetLoadKind(); + return *load_data_.ref.dex_file; +} + +// Note: defined outside class to see operator<<(., HLoadString::LoadKind). +inline uint32_t HLoadString::GetDexCacheElementOffset() const { + DCHECK(HasDexCacheReference(GetLoadKind())) << GetLoadKind(); + return load_data_.ref.dex_cache_element_index; +} + +// Note: defined outside class to see operator<<(., HLoadString::LoadKind). +inline void HLoadString::AddSpecialInput(HInstruction* special_input) { + // The special input is used for PC-relative loads on some architectures. + DCHECK(GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative || + GetLoadKind() == LoadKind::kDexCachePcRelative) << GetLoadKind(); + DCHECK(InputAt(0) == nullptr); + SetRawInputAt(0u, special_input); + special_input->AddUseAt(this, 0); +} /** * Performs an initialization check on its Class object input. diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 125c00d275..886c9e2ad4 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -259,8 +259,7 @@ class OptimizingCompiler FINAL : public Compiler { explicit OptimizingCompiler(CompilerDriver* driver); ~OptimizingCompiler(); - bool CanCompileMethod(uint32_t method_idx, const DexFile& dex_file, CompilationUnit* cu) const - OVERRIDE; + bool CanCompileMethod(uint32_t method_idx, const DexFile& dex_file) const OVERRIDE; CompiledMethod* Compile(const DexFile::CodeItem* code_item, uint32_t access_flags, @@ -283,8 +282,6 @@ class OptimizingCompiler FINAL : public Compiler { InstructionSetPointerSize(GetCompilerDriver()->GetInstructionSet()))); } - void InitCompilationUnit(CompilationUnit& cu) const OVERRIDE; - void Init() OVERRIDE; void UnInit() const OVERRIDE; @@ -365,12 +362,8 @@ OptimizingCompiler::~OptimizingCompiler() { } } -void OptimizingCompiler::InitCompilationUnit(CompilationUnit& cu ATTRIBUTE_UNUSED) const { -} - bool OptimizingCompiler::CanCompileMethod(uint32_t method_idx ATTRIBUTE_UNUSED, - const DexFile& dex_file ATTRIBUTE_UNUSED, - CompilationUnit* cu ATTRIBUTE_UNUSED) const { + const DexFile& dex_file ATTRIBUTE_UNUSED) const { return true; } @@ -662,9 +655,15 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, } DexCompilationUnit dex_compilation_unit( - nullptr, class_loader, Runtime::Current()->GetClassLinker(), dex_file, code_item, - class_def_idx, method_idx, access_flags, - nullptr, dex_cache); + class_loader, + Runtime::Current()->GetClassLinker(), + dex_file, + code_item, + class_def_idx, + method_idx, + access_flags, + /* verified_method */ nullptr, + dex_cache); bool requires_barrier = dex_compilation_unit.IsConstructor() && compiler_driver->RequiresConstructorBarrier(Thread::Current(), diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc index d281a9fc6c..dafbd3d7d1 100644 --- a/compiler/optimizing/pc_relative_fixups_x86.cc +++ b/compiler/optimizing/pc_relative_fixups_x86.cc @@ -80,6 +80,15 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { HandleInvoke(invoke); } + void VisitLoadString(HLoadString* load_string) OVERRIDE { + HLoadString::LoadKind load_kind = load_string->GetLoadKind(); + if (load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative || + load_kind == HLoadString::LoadKind::kDexCachePcRelative) { + InitializePCRelativeBasePointer(); + load_string->AddSpecialInput(base_); + } + } + void BinaryFP(HBinaryOperation* bin) { HConstant* rhs = bin->InputAt(1)->AsConstant(); if (rhs != nullptr && Primitive::IsFloatingPointType(rhs->GetType())) { diff --git a/compiler/optimizing/pretty_printer_test.cc b/compiler/optimizing/pretty_printer_test.cc index 2de0c1be72..d5b95d284a 100644 --- a/compiler/optimizing/pretty_printer_test.cc +++ b/compiler/optimizing/pretty_printer_test.cc @@ -135,13 +135,13 @@ TEST_F(PrettyPrinterTest, CFG3) { TEST_F(PrettyPrinterTest, CFG4) { const char* expected = "BasicBlock 0, succ: 3\n" - " 3: SuspendCheck\n" - " 4: Goto 3\n" + " 2: SuspendCheck\n" + " 3: Goto 3\n" "BasicBlock 1, pred: 3, 1, succ: 1\n" - " 0: SuspendCheck\n" - " 1: Goto 1\n" + " 5: SuspendCheck\n" + " 0: Goto 1\n" "BasicBlock 3, pred: 0, succ: 1\n" - " 5: Goto 1\n"; + " 4: Goto 1\n"; const uint16_t data1[] = ZERO_REGISTER_CODE_ITEM( Instruction::NOP, @@ -204,20 +204,20 @@ TEST_F(PrettyPrinterTest, CFG7) { const char* expected = "BasicBlock 0, succ: 1\n" " 1: IntConstant [5, 5]\n" - " 11: SuspendCheck\n" - " 12: Goto 1\n" + " 10: SuspendCheck\n" + " 11: Goto 1\n" "BasicBlock 1, pred: 0, succ: 5, 6\n" " 5: Equal(1, 1) [6]\n" " 6: If(5)\n" "BasicBlock 2, pred: 6, 3, succ: 3\n" " 7: Goto 3\n" "BasicBlock 3, pred: 5, 2, succ: 2\n" - " 8: SuspendCheck\n" - " 9: Goto 2\n" + " 14: SuspendCheck\n" + " 8: Goto 2\n" "BasicBlock 5, pred: 1, succ: 3\n" - " 13: Goto 3\n" + " 12: Goto 3\n" "BasicBlock 6, pred: 1, succ: 2\n" - " 14: Goto 2\n"; + " 13: Goto 2\n"; const uint16_t data[] = ONE_REGISTER_CODE_ITEM( Instruction::CONST_4 | 0 | 0, diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc index 75356c848b..95f10e0720 100644 --- a/compiler/optimizing/reference_type_propagation.cc +++ b/compiler/optimizing/reference_type_propagation.cc @@ -291,12 +291,12 @@ static bool MatchIfInstanceOf(HIf* ifInstruction, if (rhs != nullptr) { HInstruction* lhs = input->AsEqual()->GetLeastConstantLeft(); if (lhs->IsInstanceOf() && rhs->IsIntConstant()) { - if (rhs->AsIntConstant()->IsOne()) { + if (rhs->AsIntConstant()->IsTrue()) { // Case (1a) *trueBranch = ifInstruction->IfTrueSuccessor(); } else { // Case (2a) - DCHECK(rhs->AsIntConstant()->IsZero()); + DCHECK(rhs->AsIntConstant()->IsFalse()) << rhs->AsIntConstant()->GetValue(); *trueBranch = ifInstruction->IfFalseSuccessor(); } *instanceOf = lhs->AsInstanceOf(); @@ -308,12 +308,12 @@ static bool MatchIfInstanceOf(HIf* ifInstruction, if (rhs != nullptr) { HInstruction* lhs = input->AsNotEqual()->GetLeastConstantLeft(); if (lhs->IsInstanceOf() && rhs->IsIntConstant()) { - if (rhs->AsIntConstant()->IsZero()) { + if (rhs->AsIntConstant()->IsFalse()) { // Case (1b) *trueBranch = ifInstruction->IfTrueSuccessor(); } else { // Case (2b) - DCHECK(rhs->AsIntConstant()->IsOne()); + DCHECK(rhs->AsIntConstant()->IsTrue()) << rhs->AsIntConstant()->GetValue(); *trueBranch = ifInstruction->IfFalseSuccessor(); } *instanceOf = lhs->AsInstanceOf(); diff --git a/compiler/optimizing/register_allocator.cc b/compiler/optimizing/register_allocator.cc index 44bede8ac0..b1f9cbcdfa 100644 --- a/compiler/optimizing/register_allocator.cc +++ b/compiler/optimizing/register_allocator.cc @@ -1931,7 +1931,7 @@ void RegisterAllocator::Resolve() { // `GetSiblingAt` returns the sibling that contains a position, but there could be // a lifetime hole in it. `CoversSlow` returns whether the interval is live at that // position. - if (sibling->CoversSlow(block->GetLifetimeStart())) { + if ((sibling != nullptr) && sibling->CoversSlow(block->GetLifetimeStart())) { DCHECK(!sibling->HasRegister()); } } diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index 5e1d1d9954..7a1bb316e4 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -16,11 +16,20 @@ #include "sharpening.h" +#include "base/casts.h" +#include "class_linker.h" #include "code_generator.h" +#include "driver/dex_compilation_unit.h" #include "utils/dex_cache_arrays_layout-inl.h" #include "driver/compiler_driver.h" +#include "gc/heap.h" +#include "gc/space/image_space.h" +#include "handle_scope-inl.h" +#include "mirror/dex_cache.h" +#include "mirror/string.h" #include "nodes.h" #include "runtime.h" +#include "scoped_thread_state_change.h" namespace art { @@ -31,12 +40,13 @@ void HSharpening::Run() { HInstruction* instruction = it.Current(); if (instruction->IsInvokeStaticOrDirect()) { ProcessInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect()); + } else if (instruction->IsLoadString()) { + ProcessLoadString(instruction->AsLoadString()); } // TODO: Move the sharpening of invoke-virtual/-interface/-super from HGraphBuilder // here. Rewrite it to avoid the CompilerDriver's reliance on verifier data // because we know the type better when inlining. - // TODO: HLoadClass, HLoadString - select PC relative dex cache array access if - // available. + // TODO: HLoadClass - select better load kind if available. } } } @@ -143,4 +153,101 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { invoke->SetDispatchInfo(dispatch_info); } +void HSharpening::ProcessLoadString(HLoadString* load_string) { + DCHECK_EQ(load_string->GetLoadKind(), HLoadString::LoadKind::kDexCacheViaMethod); + DCHECK(!load_string->IsInDexCache()); + + const DexFile& dex_file = load_string->GetDexFile(); + uint32_t string_index = load_string->GetStringIndex(); + + bool is_in_dex_cache = false; + HLoadString::LoadKind desired_load_kind; + uint64_t address = 0u; // String or dex cache element address. + { + Runtime* runtime = Runtime::Current(); + ClassLinker* class_linker = runtime->GetClassLinker(); + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<1> hs(soa.Self()); + Handle<mirror::DexCache> dex_cache = IsSameDexFile(dex_file, *compilation_unit_.GetDexFile()) + ? compilation_unit_.GetDexCache() + : hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file)); + + if (compiler_driver_->IsBootImage()) { + // Compiling boot image. Resolve the string and allocate it if needed. + DCHECK(!runtime->UseJit()); + mirror::String* string = class_linker->ResolveString(dex_file, string_index, dex_cache); + CHECK(string != nullptr); + if (!compiler_driver_->GetSupportBootImageFixup()) { + // MIPS/MIPS64 or compiler_driver_test. Do not sharpen. + desired_load_kind = HLoadString::LoadKind::kDexCacheViaMethod; + } else { + DCHECK(ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &dex_file)); + is_in_dex_cache = true; + desired_load_kind = codegen_->GetCompilerOptions().GetCompilePic() + ? HLoadString::LoadKind::kBootImageLinkTimePcRelative + : HLoadString::LoadKind::kBootImageLinkTimeAddress; + } + } else if (runtime->UseJit()) { + // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. + // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); + mirror::String* string = dex_cache->GetResolvedString(string_index); + is_in_dex_cache = (string != nullptr); + if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) { + desired_load_kind = HLoadString::LoadKind::kBootImageAddress; + address = reinterpret_cast64<uint64_t>(string); + } else { + // Note: If the string is not in the dex cache, the instruction needs environment + // and will not be inlined across dex files. Within a dex file, the slow-path helper + // loads the correct string and inlined frames are used correctly for OOM stack trace. + // TODO: Write a test for this. + desired_load_kind = HLoadString::LoadKind::kDexCacheAddress; + void* dex_cache_element_address = &dex_cache->GetStrings()[string_index]; + address = reinterpret_cast64<uint64_t>(dex_cache_element_address); + } + } else { + // AOT app compilation. Try to lookup the string without allocating if not found. + mirror::String* string = class_linker->LookupString(dex_file, string_index, dex_cache); + if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) { + if (codegen_->GetCompilerOptions().GetCompilePic()) { + // Use PC-relative load from the dex cache if the dex file belongs + // to the oat file that we're currently compiling. + desired_load_kind = ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &dex_file) + ? HLoadString::LoadKind::kDexCachePcRelative + : HLoadString::LoadKind::kDexCacheViaMethod; + } else { + desired_load_kind = HLoadString::LoadKind::kBootImageAddress; + address = reinterpret_cast64<uint64_t>(string); + } + } else { + // Not JIT and the string is not in boot image. + desired_load_kind = HLoadString::LoadKind::kDexCachePcRelative; + } + } + } + if (is_in_dex_cache) { + load_string->MarkInDexCache(); + } + + HLoadString::LoadKind load_kind = codegen_->GetSupportedLoadStringKind(desired_load_kind); + switch (load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kDexCacheViaMethod: + load_string->SetLoadKindWithStringReference(load_kind, dex_file, string_index); + break; + case HLoadString::LoadKind::kBootImageAddress: + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK_NE(address, 0u); + load_string->SetLoadKindWithAddress(load_kind, address); + break; + case HLoadString::LoadKind::kDexCachePcRelative: { + size_t pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); + DexCacheArraysLayout layout(pointer_size, &dex_file); + size_t element_index = layout.StringOffset(string_index); + load_string->SetLoadKindWithDexCacheReference(load_kind, dex_file, element_index); + break; + } + } +} + } // namespace art diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index adae7007dd..24152f6b71 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -47,6 +47,7 @@ class HSharpening : public HOptimization { private: void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke); + void ProcessLoadString(HLoadString* load_string); CodeGenerator* codegen_; const DexCompilationUnit& compilation_unit_; diff --git a/compiler/optimizing/ssa_builder.cc b/compiler/optimizing/ssa_builder.cc index 09ca8b7b44..294d00f8e2 100644 --- a/compiler/optimizing/ssa_builder.cc +++ b/compiler/optimizing/ssa_builder.cc @@ -458,6 +458,7 @@ void SsaBuilder::RemoveRedundantUninitializedStrings() { } for (HNewInstance* new_instance : uninitialized_strings_) { + DCHECK(new_instance->IsInBlock()); // Replace NewInstance of String with NullConstant if not used prior to // calling StringFactory. In case of deoptimization, the interpreter is // expected to skip null check on the `this` argument of the StringFactory call. @@ -972,7 +973,13 @@ void SsaBuilder::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { // Replacing the NewInstance might render it redundant. Keep a list of these // to be visited once it is clear whether it is has remaining uses. if (arg_this->IsNewInstance()) { - uninitialized_strings_.push_back(arg_this->AsNewInstance()); + HNewInstance* new_instance = arg_this->AsNewInstance(); + // Note that in some rare cases (b/27847265), the same NewInstance may be seen + // multiple times. We should only consider it once for removal, so we + // ensure it is not added more than once. + if (!ContainsElement(uninitialized_strings_, new_instance)) { + uninitialized_strings_.push_back(new_instance); + } } else { DCHECK(arg_this->IsPhi()); // NewInstance is not the direct input of the StringFactory call. It might diff --git a/compiler/utils/string_reference.h b/compiler/utils/string_reference.h new file mode 100644 index 0000000000..72552f21aa --- /dev/null +++ b/compiler/utils/string_reference.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_UTILS_STRING_REFERENCE_H_ +#define ART_COMPILER_UTILS_STRING_REFERENCE_H_ + +#include <stdint.h> + +#include "base/logging.h" +#include "utf-inl.h" + +namespace art { + +class DexFile; + +// A string is uniquely located by its DexFile and the string_ids_ table index into that DexFile. +struct StringReference { + StringReference(const DexFile* file, uint32_t index) : dex_file(file), string_index(index) { } + + const DexFile* dex_file; + uint32_t string_index; +}; + +// Compare the actual referenced string values. Used for string reference deduplication. +struct StringReferenceValueComparator { + bool operator()(StringReference sr1, StringReference sr2) const { + // Note that we want to deduplicate identical strings even if they are referenced + // by different dex files, so we need some (any) total ordering of strings, rather + // than references. However, the references should usually be from the same dex file, + // so we choose the dex file string ordering so that we can simply compare indexes + // and avoid the costly string comparison in the most common case. + if (sr1.dex_file == sr2.dex_file) { + // Use the string order enforced by the dex file verifier. + DCHECK_EQ( + sr1.string_index < sr2.string_index, + CompareModifiedUtf8ToModifiedUtf8AsUtf16CodePointValues( + sr1.dex_file->GetStringData(sr1.dex_file->GetStringId(sr1.string_index)), + sr1.dex_file->GetStringData(sr2.dex_file->GetStringId(sr2.string_index))) < 0); + return sr1.string_index < sr2.string_index; + } else { + // Cannot compare indexes, so do the string comparison. + return CompareModifiedUtf8ToModifiedUtf8AsUtf16CodePointValues( + sr1.dex_file->GetStringData(sr1.dex_file->GetStringId(sr1.string_index)), + sr1.dex_file->GetStringData(sr2.dex_file->GetStringId(sr2.string_index))) < 0; + } + } +}; + +} // namespace art + +#endif // ART_COMPILER_UTILS_STRING_REFERENCE_H_ |